dataset_manipulation_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:12:59.069630Z",
     "start_time": "2018-09-07T10:12:59.065733Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:02.279658Z",
     "start_time": "2018-09-07T10:13:02.147612Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python 3.6.5 :: Anaconda, Inc.\r\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "#plt.rcParams['figure.figsize'] = [10, 5]\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import missingno as msno\n",
    "import pandas_profiling\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "!python --version\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn import cross_validation\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:03.127324Z",
     "start_time": "2018-09-07T10:13:03.123147Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "## Show all rows and columns instead of showing only some part and hiding other parts for large data.\n",
    "def show_full_data(data, row_size=None, column_size=None):\n",
    "    with pd.option_context('display.max_rows', row_size, 'display.max_columns', column_size):\n",
    "        display(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:38.270161Z",
     "start_time": "2018-09-07T10:13:05.784744Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(307511, 714)\n",
      "(48744, 713)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv('datasets/train.csv')\n",
    "test = pd.read_csv('datasets/test.csv')\n",
    "print(train.shape)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving TARGET values and Column names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:38.755215Z",
     "start_time": "2018-09-07T10:13:38.749799Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Copying labels to not lose them and use later\n",
    "train_labels = train.TARGET.copy()\n",
    "train_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:39.178257Z",
     "start_time": "2018-09-07T10:13:39.174715Z"
    }
   },
   "outputs": [],
   "source": [
    "# Saving IDs \n",
    "train_id = train.SK_ID_CURR.copy()\n",
    "test_id = test.SK_ID_CURR.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:41.033212Z",
     "start_time": "2018-09-07T10:13:39.637718Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 713)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.drop(columns='TARGET', inplace=True)\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:41.486719Z",
     "start_time": "2018-09-07T10:13:41.482827Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(713,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extracting column names to not lose them\n",
    "column_names = train.columns.copy()\n",
    "column_names.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imputing missing values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even if we dropped some features before we create our dataset, there are some missing values in our dateset. Scikit-Learn algorithms generally does not work with missing values. If we want to use scaling algorithms, the algorithm probably gives error. Also, most algorithms (lightgbm works) does not work with NaN values. Therefore, if we want to use our dataset, we should find a solution for our NaN values.  \n",
    "  \n",
    "Imputer from Scikit performs different methods to fill NaN values. I use mean strategy to calculate mean of the values and fill NaN values with this algorithm.  I do this for train and test data because if we want to make predictions on test data, test data should not have NaN values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:41.967898Z",
     "start_time": "2018-09-07T10:13:41.965552Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:49.179953Z",
     "start_time": "2018-09-07T10:13:42.337314Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ealptekin/anaconda3/envs/yasin-tf/lib/python3.6/site-packages/sklearn/preprocessing/imputation.py:347: UserWarning: Deleting features without observed values: [380 381]\n",
      "  \"observed values: %s\" % missing)\n"
     ]
    }
   ],
   "source": [
    "# Create our imputer to replace missing values with the mean e.g.\n",
    "imp = Imputer(missing_values='NaN', strategy='mean', axis=0, verbose=1)\n",
    "imp = imp.fit(train)\n",
    "# Impute our data, then train\n",
    "train_imp = imp.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:49.668398Z",
     "start_time": "2018-09-07T10:13:49.665721Z"
    }
   },
   "outputs": [],
   "source": [
    "# Choosing dropped columns during imputing.\n",
    "nanmask = np.isnan(imp.statistics_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:50.138425Z",
     "start_time": "2018-09-07T10:13:50.134510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([380, 381]),)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(nanmask==True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:50.491194Z",
     "start_time": "2018-09-07T10:13:50.488296Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['previous_app/credit_mean', 'previous_app/credit_max'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Names of dropped columns.\n",
    "column_names.values[380:382]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These 2 columns are dropped because they cannot be filled."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:50.785148Z",
     "start_time": "2018-09-07T10:13:50.782943Z"
    }
   },
   "outputs": [],
   "source": [
    "column_names_imp = column_names.values\n",
    "column_names_imp = np.delete(column_names_imp, [380, 381])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "run_control": {
     "marked": true
    }
   },
   "source": [
    "We also drop these columns from column names to equalize the size of columns and column names."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:51.108754Z",
     "start_time": "2018-09-07T10:13:51.087317Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(307511, 711)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>701</th>\n",
       "      <th>702</th>\n",
       "      <th>703</th>\n",
       "      <th>704</th>\n",
       "      <th>705</th>\n",
       "      <th>706</th>\n",
       "      <th>707</th>\n",
       "      <th>708</th>\n",
       "      <th>709</th>\n",
       "      <th>710</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-20.421053</td>\n",
       "      <td>-12.0000</td>\n",
       "      <td>-31.000000</td>\n",
       "      <td>24.257310</td>\n",
       "      <td>-388.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.160000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-14.000000</td>\n",
       "      <td>13.890000</td>\n",
       "      <td>-179.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>0.010032</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.666667</td>\n",
       "      <td>-3.0000</td>\n",
       "      <td>-11.000000</td>\n",
       "      <td>17.333333</td>\n",
       "      <td>-23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>0.008019</td>\n",
       "      <td>...</td>\n",
       "      <td>379.700182</td>\n",
       "      <td>24001.706113</td>\n",
       "      <td>-6797.926065</td>\n",
       "      <td>3.546386e+08</td>\n",
       "      <td>7133.394533</td>\n",
       "      <td>-11.130596</td>\n",
       "      <td>16.6321</td>\n",
       "      <td>-38.073301</td>\n",
       "      <td>787.185389</td>\n",
       "      <td>-353.020996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>0.028663</td>\n",
       "      <td>...</td>\n",
       "      <td>-452.384318</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-22655.655000</td>\n",
       "      <td>8.084830e+06</td>\n",
       "      <td>-29857.365000</td>\n",
       "      <td>-3.636364</td>\n",
       "      <td>12.0000</td>\n",
       "      <td>-31.000000</td>\n",
       "      <td>63.865734</td>\n",
       "      <td>-240.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 711 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0    1    2    3    4         5          6        7          8    \\\n",
       "0  100002.0  0.0  0.0  1.0  0.0  202500.0   406597.5  24700.5   351000.0   \n",
       "1  100003.0  0.0  0.0  0.0  0.0  270000.0  1293502.5  35698.5  1129500.0   \n",
       "2  100004.0  1.0  1.0  1.0  0.0   67500.0   135000.0   6750.0   135000.0   \n",
       "3  100006.0  0.0  0.0  1.0  0.0  135000.0   312682.5  29686.5   297000.0   \n",
       "4  100007.0  0.0  0.0  1.0  0.0  121500.0   513000.0  21865.5   513000.0   \n",
       "\n",
       "        9       ...             701           702           703           704  \\\n",
       "0  0.018801     ...        0.000000      0.000000      0.000000  0.000000e+00   \n",
       "1  0.003541     ...        0.000000      0.000000      0.000000  0.000000e+00   \n",
       "2  0.010032     ...        0.000000      0.000000      0.000000  0.000000e+00   \n",
       "3  0.008019     ...      379.700182  24001.706113  -6797.926065  3.546386e+08   \n",
       "4  0.028663     ...     -452.384318      0.000000 -22655.655000  8.084830e+06   \n",
       "\n",
       "            705        706      707        708         709         710  \n",
       "0      0.000000 -20.421053 -12.0000 -31.000000   24.257310 -388.000000  \n",
       "1      0.000000  -7.160000  -1.0000 -14.000000   13.890000 -179.000000  \n",
       "2      0.000000  -7.666667  -3.0000 -11.000000   17.333333  -23.000000  \n",
       "3   7133.394533 -11.130596  16.6321 -38.073301  787.185389 -353.020996  \n",
       "4 -29857.365000  -3.636364  12.0000 -31.000000   63.865734 -240.000000  \n",
       "\n",
       "[5 rows x 711 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_imp = pd.DataFrame(train_imp)\n",
    "print(train_imp.shape)\n",
    "train_imp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:51.539199Z",
     "start_time": "2018-09-07T10:13:51.518808Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_mean</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_max</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_min</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_var</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_sum</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_mean</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_max</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_min</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_var</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>351000.0</td>\n",
       "      <td>0.018801</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-20.421053</td>\n",
       "      <td>-12.0000</td>\n",
       "      <td>-31.000000</td>\n",
       "      <td>24.257310</td>\n",
       "      <td>-388.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>1129500.0</td>\n",
       "      <td>0.003541</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.160000</td>\n",
       "      <td>-1.0000</td>\n",
       "      <td>-14.000000</td>\n",
       "      <td>13.890000</td>\n",
       "      <td>-179.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>0.010032</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.666667</td>\n",
       "      <td>-3.0000</td>\n",
       "      <td>-11.000000</td>\n",
       "      <td>17.333333</td>\n",
       "      <td>-23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>297000.0</td>\n",
       "      <td>0.008019</td>\n",
       "      <td>...</td>\n",
       "      <td>379.700182</td>\n",
       "      <td>24001.706113</td>\n",
       "      <td>-6797.926065</td>\n",
       "      <td>3.546386e+08</td>\n",
       "      <td>7133.394533</td>\n",
       "      <td>-11.130596</td>\n",
       "      <td>16.6321</td>\n",
       "      <td>-38.073301</td>\n",
       "      <td>787.185389</td>\n",
       "      <td>-353.020996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>0.028663</td>\n",
       "      <td>...</td>\n",
       "      <td>-452.384318</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-22655.655000</td>\n",
       "      <td>8.084830e+06</td>\n",
       "      <td>-29857.365000</td>\n",
       "      <td>-3.636364</td>\n",
       "      <td>12.0000</td>\n",
       "      <td>-31.000000</td>\n",
       "      <td>63.865734</td>\n",
       "      <td>-240.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 711 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  NAME_CONTRACT_TYPE  FLAG_OWN_CAR  FLAG_OWN_REALTY  \\\n",
       "0    100002.0                 0.0           0.0              1.0   \n",
       "1    100003.0                 0.0           0.0              0.0   \n",
       "2    100004.0                 1.0           1.0              1.0   \n",
       "3    100006.0                 0.0           0.0              1.0   \n",
       "4    100007.0                 0.0           0.0              1.0   \n",
       "\n",
       "   CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  AMT_GOODS_PRICE  \\\n",
       "0           0.0          202500.0    406597.5      24700.5         351000.0   \n",
       "1           0.0          270000.0   1293502.5      35698.5        1129500.0   \n",
       "2           0.0           67500.0    135000.0       6750.0         135000.0   \n",
       "3           0.0          135000.0    312682.5      29686.5         297000.0   \n",
       "4           0.0          121500.0    513000.0      21865.5         513000.0   \n",
       "\n",
       "   REGION_POPULATION_RELATIVE                    ...                      \\\n",
       "0                    0.018801                    ...                       \n",
       "1                    0.003541                    ...                       \n",
       "2                    0.010032                    ...                       \n",
       "3                    0.008019                    ...                       \n",
       "4                    0.028663                    ...                       \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_mean  \\\n",
       "0                                   0.000000   \n",
       "1                                   0.000000   \n",
       "2                                   0.000000   \n",
       "3                                 379.700182   \n",
       "4                                -452.384318   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_max  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000000   \n",
       "2                                  0.000000   \n",
       "3                              24001.706113   \n",
       "4                                  0.000000   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_min  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000000   \n",
       "2                                  0.000000   \n",
       "3                              -6797.926065   \n",
       "4                             -22655.655000   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_var  \\\n",
       "0                              0.000000e+00   \n",
       "1                              0.000000e+00   \n",
       "2                              0.000000e+00   \n",
       "3                              3.546386e+08   \n",
       "4                              8.084830e+06   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_sum  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000000   \n",
       "2                                  0.000000   \n",
       "3                               7133.394533   \n",
       "4                             -29857.365000   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_mean  \\\n",
       "0                                  -20.421053   \n",
       "1                                   -7.160000   \n",
       "2                                   -7.666667   \n",
       "3                                  -11.130596   \n",
       "4                                   -3.636364   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_max  \\\n",
       "0                                   -12.0000   \n",
       "1                                    -1.0000   \n",
       "2                                    -3.0000   \n",
       "3                                    16.6321   \n",
       "4                                    12.0000   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_min  \\\n",
       "0                                 -31.000000   \n",
       "1                                 -14.000000   \n",
       "2                                 -11.000000   \n",
       "3                                 -38.073301   \n",
       "4                                 -31.000000   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_var  \\\n",
       "0                                  24.257310   \n",
       "1                                  13.890000   \n",
       "2                                  17.333333   \n",
       "3                                 787.185389   \n",
       "4                                  63.865734   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_sum  \n",
       "0                                -388.000000  \n",
       "1                                -179.000000  \n",
       "2                                 -23.000000  \n",
       "3                                -353.020996  \n",
       "4                                -240.000000  \n",
       "\n",
       "[5 rows x 711 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_imp.columns = column_names_imp\n",
    "train_imp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:55.212222Z",
     "start_time": "2018-09-07T10:13:51.977593Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_imp.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-09-01T20:34:16.277Z"
    }
   },
   "source": [
    "We checked how many null values in our dataset. It is zero so we have completed our goal."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We are doing similar process for test data to fill all NaN values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:56.562481Z",
     "start_time": "2018-09-07T10:13:55.812505Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48744, 713)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp = imp.fit(test)\n",
    "# Impute our data, then train\n",
    "test_imp = imp.transform(test)\n",
    "test_imp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:57.219414Z",
     "start_time": "2018-09-07T10:13:57.215320Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([], dtype=int64),)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nanmask_test = np.isnan(imp.statistics_)\n",
    "np.where(nanmask_test==True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:58.150289Z",
     "start_time": "2018-09-07T10:13:57.901661Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48744, 711)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_imp = np.delete(test_imp, [380, 381], axis=1)\n",
    "test_imp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:58.799352Z",
     "start_time": "2018-09-07T10:13:58.775997Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_mean</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_max</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_min</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_var</th>\n",
       "      <th>installments_AMT_PAYMENT-INSTALLMENT_sum</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_mean</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_max</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_min</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_var</th>\n",
       "      <th>installments_DAYS_PAYMENT-INSTALLMENT_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>568800.0</td>\n",
       "      <td>20560.5</td>\n",
       "      <td>450000.0</td>\n",
       "      <td>0.018850</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-7.285714</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>-36.000000</td>\n",
       "      <td>213.904762</td>\n",
       "      <td>-51.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100005.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>99000.0</td>\n",
       "      <td>222768.0</td>\n",
       "      <td>17370.0</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>0.035792</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-23.555556</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-37.000000</td>\n",
       "      <td>182.527778</td>\n",
       "      <td>-212.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100013.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>663264.0</td>\n",
       "      <td>69777.0</td>\n",
       "      <td>630000.0</td>\n",
       "      <td>0.019101</td>\n",
       "      <td>...</td>\n",
       "      <td>-1157.662742</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-23147.820000</td>\n",
       "      <td>2.346772e+07</td>\n",
       "      <td>-179437.725000</td>\n",
       "      <td>-5.180645</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>-38.000000</td>\n",
       "      <td>127.200922</td>\n",
       "      <td>-803.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100028.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>315000.0</td>\n",
       "      <td>1575000.0</td>\n",
       "      <td>49018.5</td>\n",
       "      <td>1575000.0</td>\n",
       "      <td>0.026392</td>\n",
       "      <td>...</td>\n",
       "      <td>-622.550708</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-8505.000000</td>\n",
       "      <td>2.972476e+06</td>\n",
       "      <td>-70348.230000</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>-19.000000</td>\n",
       "      <td>26.678571</td>\n",
       "      <td>-339.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100038.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>625500.0</td>\n",
       "      <td>32067.0</td>\n",
       "      <td>625500.0</td>\n",
       "      <td>0.010032</td>\n",
       "      <td>...</td>\n",
       "      <td>571.951414</td>\n",
       "      <td>28566.898614</td>\n",
       "      <td>-7136.377516</td>\n",
       "      <td>3.803835e+08</td>\n",
       "      <td>15742.302464</td>\n",
       "      <td>-11.552455</td>\n",
       "      <td>12.531942</td>\n",
       "      <td>-38.759448</td>\n",
       "      <td>544.772430</td>\n",
       "      <td>-381.936021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 711 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  NAME_CONTRACT_TYPE  FLAG_OWN_CAR  FLAG_OWN_REALTY  \\\n",
       "0    100001.0                 0.0           0.0              1.0   \n",
       "1    100005.0                 0.0           0.0              1.0   \n",
       "2    100013.0                 0.0           1.0              1.0   \n",
       "3    100028.0                 0.0           0.0              1.0   \n",
       "4    100038.0                 0.0           1.0              0.0   \n",
       "\n",
       "   CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  AMT_GOODS_PRICE  \\\n",
       "0           0.0          135000.0    568800.0      20560.5         450000.0   \n",
       "1           0.0           99000.0    222768.0      17370.0         180000.0   \n",
       "2           0.0          202500.0    663264.0      69777.0         630000.0   \n",
       "3           2.0          315000.0   1575000.0      49018.5        1575000.0   \n",
       "4           1.0          180000.0    625500.0      32067.0         625500.0   \n",
       "\n",
       "   REGION_POPULATION_RELATIVE                    ...                      \\\n",
       "0                    0.018850                    ...                       \n",
       "1                    0.035792                    ...                       \n",
       "2                    0.019101                    ...                       \n",
       "3                    0.026392                    ...                       \n",
       "4                    0.010032                    ...                       \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_mean  \\\n",
       "0                                   0.000000   \n",
       "1                                   0.000000   \n",
       "2                               -1157.662742   \n",
       "3                                -622.550708   \n",
       "4                                 571.951414   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_max  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000000   \n",
       "2                                  0.000000   \n",
       "3                                  0.000000   \n",
       "4                              28566.898614   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_min  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000000   \n",
       "2                             -23147.820000   \n",
       "3                              -8505.000000   \n",
       "4                              -7136.377516   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_var  \\\n",
       "0                              0.000000e+00   \n",
       "1                              0.000000e+00   \n",
       "2                              2.346772e+07   \n",
       "3                              2.972476e+06   \n",
       "4                              3.803835e+08   \n",
       "\n",
       "   installments_AMT_PAYMENT-INSTALLMENT_sum  \\\n",
       "0                                  0.000000   \n",
       "1                                  0.000000   \n",
       "2                            -179437.725000   \n",
       "3                             -70348.230000   \n",
       "4                              15742.302464   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_mean  \\\n",
       "0                                   -7.285714   \n",
       "1                                  -23.555556   \n",
       "2                                   -5.180645   \n",
       "3                                   -3.000000   \n",
       "4                                  -11.552455   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_max  \\\n",
       "0                                  11.000000   \n",
       "1                                   1.000000   \n",
       "2                                  21.000000   \n",
       "3                                   7.000000   \n",
       "4                                  12.531942   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_min  \\\n",
       "0                                 -36.000000   \n",
       "1                                 -37.000000   \n",
       "2                                 -38.000000   \n",
       "3                                 -19.000000   \n",
       "4                                 -38.759448   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_var  \\\n",
       "0                                 213.904762   \n",
       "1                                 182.527778   \n",
       "2                                 127.200922   \n",
       "3                                  26.678571   \n",
       "4                                 544.772430   \n",
       "\n",
       "   installments_DAYS_PAYMENT-INSTALLMENT_sum  \n",
       "0                                 -51.000000  \n",
       "1                                -212.000000  \n",
       "2                                -803.000000  \n",
       "3                                -339.000000  \n",
       "4                                -381.936021  \n",
       "\n",
       "[5 rows x 711 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_imp = pd.DataFrame(test_imp)\n",
    "test_imp.columns = column_names_imp\n",
    "test_imp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:13:59.743925Z",
     "start_time": "2018-09-07T10:13:59.352971Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_imp.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:14:02.159330Z",
     "start_time": "2018-09-07T10:14:00.235390Z"
    }
   },
   "outputs": [],
   "source": [
    "train_imp = train_imp.drop(columns='SK_ID_CURR')\n",
    "test_imp = test_imp.drop(columns='SK_ID_CURR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:14:02.726116Z",
     "start_time": "2018-09-07T10:14:02.724358Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# train_imp.to_csv('datasets/train_imp.csv', index=False)\n",
    "# test_imp.to_csv('datasets/test_imp.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sampling Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- There are a lot of models for classification problems. Some models like deep neural networks have high performance but they need high performance computers. In this problem, because of I am using a laptop to make computations, I would like to try less computationally expensive model.\n",
    "- Moreover, we have imbalanced dataset; one class has way less instances than other one. Risky instances (TARGET=1) are rare instances in our dataset. There are different approaches like undersampling and oversampling to overcome this problem but none of them increases performance too much. In this type of problem, if we can reach the data source, we should try to collect more data for rare class. But, in this work, we cannot reach the data source and we try to increase the size of rare instances. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are a lot of different ranges of data in our dataset. Some of them are too large and some of them are too small. Calculations with these types of high range of numbers could be hard for computers. Relating values with each other could be hard and finding similarities in high dimensions could not be reachable. Moreover, if we want to use neural networks, we should normalize our values to increase performance of neural networks. Therefore, before we move to feature selection and modeling parts, I would like to scale feature values in between 0 and 1.   \n",
    "  \n",
    "This scaling could be better with transformations. For instance, we have some skewed data which have a lot of values in a short range on one side of distribution. When we scale highly skewed data, it would not have distinguishing values. However, I do not transform my data to see the results. \n",
    "  \n",
    "I oversampled to increase the size of rare data and undersampled to decrease the size of common data. I use SMOTE method which is highly effective for oversampling. For undersampling, I use Random Under Sampler."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:14:03.383222Z",
     "start_time": "2018-09-07T10:14:03.329409Z"
    }
   },
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.under_sampling import RandomUnderSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:14:04.993660Z",
     "start_time": "2018-09-07T10:14:03.882893Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number transactions X_train dataset:  (307511, 710)\n",
      "Number transactions y_train dataset:  (48744, 710)\n",
      "Before OverSampling, counts of label '0': 282686\n",
      "Before OverSampling, counts of label '1': 24825 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Number transactions X_train dataset: \", train_imp.shape)\n",
    "print(\"Number transactions y_train dataset: \", test_imp.shape)\n",
    "\n",
    "print(\"Before OverSampling, counts of label '0': {}\".format(sum(train_labels.values==0)))\n",
    "print(\"Before OverSampling, counts of label '1': {} \\n\".format(sum(train_labels.values==1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:15:03.471704Z",
     "start_time": "2018-09-07T10:14:05.658640Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After OverSampling, the shape of train_X: (565372, 710)\n",
      "After OverSampling, the shape of train_y: (565372,) \n",
      "\n",
      "After OverSampling, counts of label '0': 282686\n",
      "After OverSampling, counts of label '1': 282686\n"
     ]
    }
   ],
   "source": [
    "sm = SMOTE()\n",
    "train_os, label_os = sm.fit_sample(train_imp, train_labels.values.ravel())\n",
    "\n",
    "print('After OverSampling, the shape of train_X: {}'.format(train_os.shape))\n",
    "print('After OverSampling, the shape of train_y: {} \\n'.format(label_os.shape))\n",
    "print(\"After OverSampling, counts of label '0': {}\".format(sum(label_os==0)))\n",
    "print(\"After OverSampling, counts of label '1': {}\".format(sum(label_os==1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:15:06.447972Z",
     "start_time": "2018-09-07T10:15:04.005364Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After Undersampling, the shape of train_X: (49650, 710)\n",
      "After Undersampling, the shape of train_y: (49650,) \n",
      "\n",
      "After Undersampling, counts of label '0': 24825\n",
      "After Undersampling, counts of label '1': 24825\n"
     ]
    }
   ],
   "source": [
    "rus = RandomUnderSampler(return_indices=True)\n",
    "train_rus, label_rus, idx_resampled = rus.fit_sample(train_imp, train_labels.values.ravel())\n",
    "\n",
    "print('After Undersampling, the shape of train_X: {}'.format(train_rus.shape))\n",
    "print('After Undersampling, the shape of train_y: {} \\n'.format(label_rus.shape))\n",
    "print(\"After Undersampling, counts of label '0': {}\".format(sum(label_rus==0)))\n",
    "print(\"After Undersampling, counts of label '1': {}\".format(sum(label_rus==1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:15:06.958291Z",
     "start_time": "2018-09-07T10:15:06.956541Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_os.out', train_os, delimiter=',')\n",
    "# np.savetxt('datasets/label_os.out', label_os, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-02T12:02:09.545385Z",
     "start_time": "2018-09-02T12:02:09.542253Z"
    }
   },
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We created a lot of new features during dataset creation. However, some of these features may not be useful for training machine learning model. They may decrease the performance or increase the chance of overfitting. Therefore, we can use feature selection techniques to eliminate unnecessary features. There are different types of approaches , some of them needs too much computing power and others may not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T02:15:14.912480Z",
     "start_time": "2018-09-07T02:15:14.908681Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import mutual_info_classif, chi2, f_classif"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Oversampled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:14.551778Z",
     "start_time": "2018-09-07T02:15:14.913871Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SelectKBest(k=250,\n",
       "      score_func=<function mutual_info_classif at 0x7f0149c23840>)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selector_os = SelectKBest(mutual_info_classif, k=250)\n",
    "selector_os.fit(train_os, label_os)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:18.773518Z",
     "start_time": "2018-09-07T04:03:14.553842Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(565372, 250)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_os_sel = selector_os.transform(train_os)\n",
    "train_os_sel.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:18.777954Z",
     "start_time": "2018-09-07T04:03:18.775401Z"
    }
   },
   "outputs": [],
   "source": [
    "# selector_os.get_support(indices=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:18.782730Z",
     "start_time": "2018-09-07T04:03:18.779576Z"
    }
   },
   "outputs": [],
   "source": [
    "# selected_features_os = selector_os.get_support(indices=True)\n",
    "# selected_features_os = pd.DataFrame(selected_features_os)\n",
    "# selected_features_os.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:18.788048Z",
     "start_time": "2018-09-07T04:03:18.784406Z"
    }
   },
   "outputs": [],
   "source": [
    "#selected_features_os.to_csv('selected_features_os_250.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:18.920157Z",
     "start_time": "2018-09-07T04:03:18.789547Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48744, 250)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_os_sel = selector_os.transform(test_imp)\n",
    "test_os_sel.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:03:18.924021Z",
     "start_time": "2018-09-07T04:03:18.921967Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_os_sel.out', train_os_sel, delimiter=',')\n",
    "# np.savetxt('datasets/test_os_sel.out', test_os_sel, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Undersampled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.097682Z",
     "start_time": "2018-09-07T04:03:18.925503Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49650, 250)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selector_rus = SelectKBest(mutual_info_classif, k=250)\n",
    "selector_rus.fit(train_rus, label_rus)\n",
    "train_rus_sel = selector_rus.transform(train_rus)\n",
    "train_rus_sel.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.102458Z",
     "start_time": "2018-09-07T04:09:04.099797Z"
    }
   },
   "outputs": [],
   "source": [
    "# selector_rus.get_support(indices=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.106975Z",
     "start_time": "2018-09-07T04:09:04.103981Z"
    }
   },
   "outputs": [],
   "source": [
    "# selected_features_rus = selector_rus.get_support(indices=True)\n",
    "# selected_features_rus = pd.DataFrame(selected_features_rus)\n",
    "# selected_features_rus.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.111034Z",
     "start_time": "2018-09-07T04:09:04.108586Z"
    }
   },
   "outputs": [],
   "source": [
    "# selected_features_rus.to_csv('selected_features_rus_250.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.250176Z",
     "start_time": "2018-09-07T04:09:04.112777Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48744, 250)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_rus_sel = selector_rus.transform(test_imp)\n",
    "test_rus_sel.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.254036Z",
     "start_time": "2018-09-07T04:09:04.251900Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_rus_sel.out', train_rus_sel, delimiter=',')\n",
    "# np.savetxt('datasets/test_rus_sel.out', test_rus_sel, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explain PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:04.258317Z",
     "start_time": "2018-09-07T04:09:04.255468Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Oversampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:05.182912Z",
     "start_time": "2018-09-07T04:09:04.259924Z"
    }
   },
   "outputs": [],
   "source": [
    "concat_os = np.concatenate((train_os_sel, test_os_sel), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:16.369908Z",
     "start_time": "2018-09-07T04:09:05.186008Z"
    }
   },
   "outputs": [],
   "source": [
    "pca_os = PCA(n_components=50)\n",
    "pca_os.fit(concat_os)\n",
    "concat_os_pca = pca_os.transform(concat_os)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:16.375377Z",
     "start_time": "2018-09-07T04:09:16.372377Z"
    }
   },
   "outputs": [],
   "source": [
    "train_os_pca = concat_os_pca[:565372, :]\n",
    "test_os_pca = concat_os_pca[565372:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:16.381354Z",
     "start_time": "2018-09-07T04:09:16.376973Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(565372, 50)\n",
      "(48744, 50)\n"
     ]
    }
   ],
   "source": [
    "print(train_os_pca.shape)\n",
    "print(test_os_pca.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:16.385205Z",
     "start_time": "2018-09-07T04:09:16.382846Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_os_pca.out', train_os_pca, delimiter=',')\n",
    "# np.savetxt('datasets/test_os_pca.out', test_os_pca, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Undersampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:16.539958Z",
     "start_time": "2018-09-07T04:09:16.386646Z"
    }
   },
   "outputs": [],
   "source": [
    "concat_rus = np.concatenate((train_rus_sel, test_rus_sel), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:26.005962Z",
     "start_time": "2018-09-07T04:09:16.541772Z"
    }
   },
   "outputs": [],
   "source": [
    "pca_rus = PCA(n_components=50)\n",
    "pca_rus.fit(concat_os)\n",
    "concat_rus_pca = pca_rus.transform(concat_rus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:26.011498Z",
     "start_time": "2018-09-07T04:09:26.008489Z"
    }
   },
   "outputs": [],
   "source": [
    "train_rus_pca = concat_rus_pca[:49650, :]\n",
    "test_rus_pca = concat_rus_pca[49650:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:26.018229Z",
     "start_time": "2018-09-07T04:09:26.013035Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49650, 50)\n",
      "(48744, 50)\n"
     ]
    }
   ],
   "source": [
    "print(train_rus_pca.shape)\n",
    "print(test_rus_pca.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:26.022353Z",
     "start_time": "2018-09-07T04:09:26.019804Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_rus_pca.out', train_rus_pca, delimiter=',')\n",
    "# np.savetxt('datasets/test_rus_pca.out', test_rus_pca, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:26.026847Z",
     "start_time": "2018-09-07T04:09:26.023922Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import RobustScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Oversampled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:26.031417Z",
     "start_time": "2018-09-07T04:09:26.028426Z"
    }
   },
   "outputs": [],
   "source": [
    "scaler = RobustScaler(with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0), copy=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:32.946612Z",
     "start_time": "2018-09-07T04:09:26.032982Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RobustScaler(copy=True, quantile_range=(25.0, 75.0), with_centering=True,\n",
       "       with_scaling=True)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler.fit(train_os_sel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:33.694782Z",
     "start_time": "2018-09-07T04:09:32.948362Z"
    }
   },
   "outputs": [],
   "source": [
    "train_os_sel_scaled = scaler.transform(train_os_sel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:33.718784Z",
     "start_time": "2018-09-07T04:09:33.696590Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>240</th>\n",
       "      <th>241</th>\n",
       "      <th>242</th>\n",
       "      <th>243</th>\n",
       "      <th>244</th>\n",
       "      <th>245</th>\n",
       "      <th>246</th>\n",
       "      <th>247</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.627962</td>\n",
       "      <td>-0.220674</td>\n",
       "      <td>-0.026348</td>\n",
       "      <td>-0.239638</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.881637</td>\n",
       "      <td>0.283086</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.270597</td>\n",
       "      <td>-0.472836</td>\n",
       "      <td>0.521425</td>\n",
       "      <td>0.070630</td>\n",
       "      <td>0.046056</td>\n",
       "      <td>-0.140886</td>\n",
       "      <td>-1.584293</td>\n",
       "      <td>-1.082245</td>\n",
       "      <td>0.410810</td>\n",
       "      <td>-0.315520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.226408</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.377962</td>\n",
       "      <td>1.700038</td>\n",
       "      <td>0.682014</td>\n",
       "      <td>1.644790</td>\n",
       "      <td>-0.969528</td>\n",
       "      <td>-0.265606</td>\n",
       "      <td>0.003049</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.300571</td>\n",
       "      <td>-0.195648</td>\n",
       "      <td>-0.537656</td>\n",
       "      <td>-0.423779</td>\n",
       "      <td>-0.141426</td>\n",
       "      <td>-0.234316</td>\n",
       "      <td>0.677102</td>\n",
       "      <td>-0.420873</td>\n",
       "      <td>1.807564</td>\n",
       "      <td>0.588405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.107354</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.872038</td>\n",
       "      <td>-0.808855</td>\n",
       "      <td>-1.182510</td>\n",
       "      <td>-0.762485</td>\n",
       "      <td>-0.557129</td>\n",
       "      <td>-0.623884</td>\n",
       "      <td>0.492479</td>\n",
       "      <td>...</td>\n",
       "      <td>0.395489</td>\n",
       "      <td>-1.212005</td>\n",
       "      <td>-1.200658</td>\n",
       "      <td>-1.059448</td>\n",
       "      <td>-0.217198</td>\n",
       "      <td>-0.413761</td>\n",
       "      <td>0.590701</td>\n",
       "      <td>-0.541122</td>\n",
       "      <td>2.054050</td>\n",
       "      <td>1.263106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.122038</td>\n",
       "      <td>-0.424059</td>\n",
       "      <td>0.294791</td>\n",
       "      <td>-0.370350</td>\n",
       "      <td>-0.685023</td>\n",
       "      <td>-0.617444</td>\n",
       "      <td>-0.937692</td>\n",
       "      <td>...</td>\n",
       "      <td>0.689048</td>\n",
       "      <td>0.189297</td>\n",
       "      <td>0.489747</td>\n",
       "      <td>0.469461</td>\n",
       "      <td>0.853128</td>\n",
       "      <td>0.709366</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.639252</td>\n",
       "      <td>-0.170347</td>\n",
       "      <td>-0.164236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.272038</td>\n",
       "      <td>0.009755</td>\n",
       "      <td>-0.208946</td>\n",
       "      <td>0.152497</td>\n",
       "      <td>0.626572</td>\n",
       "      <td>-0.763049</td>\n",
       "      <td>-0.937184</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.060844</td>\n",
       "      <td>2.160453</td>\n",
       "      <td>-0.114572</td>\n",
       "      <td>-0.070630</td>\n",
       "      <td>-0.081979</td>\n",
       "      <td>0.266943</td>\n",
       "      <td>1.277985</td>\n",
       "      <td>0.360748</td>\n",
       "      <td>0.410810</td>\n",
       "      <td>0.324580</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 250 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0         1    2         3         4         5         6         7    \\\n",
       "0  0.000000  0.000000  0.0  0.627962 -0.220674 -0.026348 -0.239638  0.000000   \n",
       "1  0.000000 -1.226408  0.0  1.377962  1.700038  0.682014  1.644790 -0.969528   \n",
       "2  1.107354  0.000000  0.0 -0.872038 -0.808855 -1.182510 -0.762485 -0.557129   \n",
       "3  0.000000  0.000000  0.0 -0.122038 -0.424059  0.294791 -0.370350 -0.685023   \n",
       "4  0.000000  0.000000  0.0 -0.272038  0.009755 -0.208946  0.152497  0.626572   \n",
       "\n",
       "        8         9      ...          240       241       242       243  \\\n",
       "0  0.881637  0.283086    ...    -0.270597 -0.472836  0.521425  0.070630   \n",
       "1 -0.265606  0.003049    ...    -0.300571 -0.195648 -0.537656 -0.423779   \n",
       "2 -0.623884  0.492479    ...     0.395489 -1.212005 -1.200658 -1.059448   \n",
       "3 -0.617444 -0.937692    ...     0.689048  0.189297  0.489747  0.469461   \n",
       "4 -0.763049 -0.937184    ...    -0.060844  2.160453 -0.114572 -0.070630   \n",
       "\n",
       "        244       245       246       247       248       249  \n",
       "0  0.046056 -0.140886 -1.584293 -1.082245  0.410810 -0.315520  \n",
       "1 -0.141426 -0.234316  0.677102 -0.420873  1.807564  0.588405  \n",
       "2 -0.217198 -0.413761  0.590701 -0.541122  2.054050  1.263106  \n",
       "3  0.853128  0.709366  0.000000  0.639252 -0.170347 -0.164236  \n",
       "4 -0.081979  0.266943  1.277985  0.360748  0.410810  0.324580  \n",
       "\n",
       "[5 rows x 250 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_os_sel_scaled = pd.DataFrame(train_os_sel_scaled)\n",
    "train_os_sel_scaled.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:34.291269Z",
     "start_time": "2018-09-07T04:09:33.720206Z"
    }
   },
   "outputs": [],
   "source": [
    "scaler_test = RobustScaler(with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0), copy=True)\n",
    "scaler_test.fit(test_os_sel)\n",
    "test_os_sel_scaled = scaler_test.transform(test_os_sel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:34.313366Z",
     "start_time": "2018-09-07T04:09:34.293045Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>240</th>\n",
       "      <th>241</th>\n",
       "      <th>242</th>\n",
       "      <th>243</th>\n",
       "      <th>244</th>\n",
       "      <th>245</th>\n",
       "      <th>246</th>\n",
       "      <th>247</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0.286707</td>\n",
       "      <td>-0.291541</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.483966</td>\n",
       "      <td>-0.396327</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.031154</td>\n",
       "      <td>-0.785714</td>\n",
       "      <td>-0.792839</td>\n",
       "      <td>-0.772730</td>\n",
       "      <td>-0.147964</td>\n",
       "      <td>-0.291908</td>\n",
       "      <td>0.535583</td>\n",
       "      <td>0.591194</td>\n",
       "      <td>-0.066667</td>\n",
       "      <td>0.897690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.52</td>\n",
       "      <td>-0.548393</td>\n",
       "      <td>-0.455852</td>\n",
       "      <td>-0.533333</td>\n",
       "      <td>0.908077</td>\n",
       "      <td>-0.319143</td>\n",
       "      <td>-1.214996</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.081504</td>\n",
       "      <td>-0.714286</td>\n",
       "      <td>-0.371762</td>\n",
       "      <td>-0.450759</td>\n",
       "      <td>-0.098401</td>\n",
       "      <td>-0.258326</td>\n",
       "      <td>-1.752249</td>\n",
       "      <td>-0.147798</td>\n",
       "      <td>-0.133333</td>\n",
       "      <td>0.366337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.514683</td>\n",
       "      <td>2.243105</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>0.013453</td>\n",
       "      <td>-0.595575</td>\n",
       "      <td>-1.210788</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103462</td>\n",
       "      <td>0.464286</td>\n",
       "      <td>6.762943</td>\n",
       "      <td>6.246231</td>\n",
       "      <td>9.889809</td>\n",
       "      <td>8.438212</td>\n",
       "      <td>0.831593</td>\n",
       "      <td>1.330186</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>-1.584158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.40</td>\n",
       "      <td>2.715030</td>\n",
       "      <td>1.174044</td>\n",
       "      <td>2.911111</td>\n",
       "      <td>0.404245</td>\n",
       "      <td>0.253326</td>\n",
       "      <td>-0.219204</td>\n",
       "      <td>...</td>\n",
       "      <td>0.791184</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>4.326692</td>\n",
       "      <td>3.734860</td>\n",
       "      <td>3.625167</td>\n",
       "      <td>4.135796</td>\n",
       "      <td>1.138231</td>\n",
       "      <td>0.295597</td>\n",
       "      <td>1.066667</td>\n",
       "      <td>-0.052805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.423545</td>\n",
       "      <td>0.301043</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>-0.472638</td>\n",
       "      <td>0.384400</td>\n",
       "      <td>-0.343535</td>\n",
       "      <td>...</td>\n",
       "      <td>0.787565</td>\n",
       "      <td>0.283725</td>\n",
       "      <td>0.615957</td>\n",
       "      <td>0.613635</td>\n",
       "      <td>0.908264</td>\n",
       "      <td>0.784298</td>\n",
       "      <td>-0.064398</td>\n",
       "      <td>0.704403</td>\n",
       "      <td>-0.250630</td>\n",
       "      <td>-0.194508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 250 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0    1    2     3         4         5         6         7         8    \\\n",
       "0  0.0  0.0  0.0 -0.20  0.286707 -0.291541  0.133333  0.000000 -0.483966   \n",
       "1  0.0  0.0  0.0 -0.52 -0.548393 -0.455852 -0.533333  0.908077 -0.319143   \n",
       "2  1.0  0.0  0.0  0.40  0.514683  2.243105  0.577778  0.013453 -0.595575   \n",
       "3  0.0  0.0  2.0  1.40  2.715030  1.174044  2.911111  0.404245  0.253326   \n",
       "4  1.0 -1.0  1.0  0.20  0.423545  0.301043  0.566667 -0.472638  0.384400   \n",
       "\n",
       "        9      ...          240       241       242       243       244  \\\n",
       "0 -0.396327    ...    -0.031154 -0.785714 -0.792839 -0.772730 -0.147964   \n",
       "1 -1.214996    ...    -0.081504 -0.714286 -0.371762 -0.450759 -0.098401   \n",
       "2 -1.210788    ...     0.103462  0.464286  6.762943  6.246231  9.889809   \n",
       "3 -0.219204    ...     0.791184  0.785714  4.326692  3.734860  3.625167   \n",
       "4 -0.343535    ...     0.787565  0.283725  0.615957  0.613635  0.908264   \n",
       "\n",
       "        245       246       247       248       249  \n",
       "0 -0.291908  0.535583  0.591194 -0.066667  0.897690  \n",
       "1 -0.258326 -1.752249 -0.147798 -0.133333  0.366337  \n",
       "2  8.438212  0.831593  1.330186 -0.200000 -1.584158  \n",
       "3  4.135796  1.138231  0.295597  1.066667 -0.052805  \n",
       "4  0.784298 -0.064398  0.704403 -0.250630 -0.194508  \n",
       "\n",
       "[5 rows x 250 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_os_sel_scaled = pd.DataFrame(test_os_sel_scaled)\n",
    "test_os_sel_scaled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:34.316799Z",
     "start_time": "2018-09-07T04:09:34.315031Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_os_sel_scaled.out', train_os_sel_scaled, delimiter=',')\n",
    "# np.savetxt('datasets/test_os_sel_scaled.out', test_os_sel_scaled, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## For Undersampled Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:34.887377Z",
     "start_time": "2018-09-07T04:09:34.318435Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>240</th>\n",
       "      <th>241</th>\n",
       "      <th>242</th>\n",
       "      <th>243</th>\n",
       "      <th>244</th>\n",
       "      <th>245</th>\n",
       "      <th>246</th>\n",
       "      <th>247</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.518822e+05</td>\n",
       "      <td>20979.0</td>\n",
       "      <td>3.377962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.551508</td>\n",
       "      <td>-1.091257</td>\n",
       "      <td>-107246.452668</td>\n",
       "      <td>1.738143</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>13668.364853</td>\n",
       "      <td>264.722662</td>\n",
       "      <td>1238.200538</td>\n",
       "      <td>-1.265235e+00</td>\n",
       "      <td>296.464297</td>\n",
       "      <td>-0.422659</td>\n",
       "      <td>1.898090</td>\n",
       "      <td>-0.951974</td>\n",
       "      <td>2.875669</td>\n",
       "      <td>1.107406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.778954e+05</td>\n",
       "      <td>15862.5</td>\n",
       "      <td>0.977962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.196168</td>\n",
       "      <td>-1.094868</td>\n",
       "      <td>-570408.734682</td>\n",
       "      <td>2.106473</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>43932.974950</td>\n",
       "      <td>6530.528782</td>\n",
       "      <td>1469.718291</td>\n",
       "      <td>1.949371e+08</td>\n",
       "      <td>5892.504535</td>\n",
       "      <td>0.140442</td>\n",
       "      <td>1218.349279</td>\n",
       "      <td>-1.029972</td>\n",
       "      <td>4.324358</td>\n",
       "      <td>-0.164236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.107354</td>\n",
       "      <td>2.114595e+06</td>\n",
       "      <td>50544.0</td>\n",
       "      <td>13.377962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.523555</td>\n",
       "      <td>-1.093467</td>\n",
       "      <td>-461829.264180</td>\n",
       "      <td>1.616099</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>79464.752034</td>\n",
       "      <td>15362.335289</td>\n",
       "      <td>3.328419</td>\n",
       "      <td>1.339639e+08</td>\n",
       "      <td>22997.734839</td>\n",
       "      <td>-0.422659</td>\n",
       "      <td>1.898090</td>\n",
       "      <td>-0.734273</td>\n",
       "      <td>2.957831</td>\n",
       "      <td>-0.786946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.317482e+05</td>\n",
       "      <td>13401.0</td>\n",
       "      <td>1.127962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.780802</td>\n",
       "      <td>-1.116898</td>\n",
       "      <td>-331457.658872</td>\n",
       "      <td>2.116840</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>65220.968011</td>\n",
       "      <td>9635.096700</td>\n",
       "      <td>1868.980047</td>\n",
       "      <td>1.833251e+08</td>\n",
       "      <td>3538.963401</td>\n",
       "      <td>-0.422659</td>\n",
       "      <td>1.898090</td>\n",
       "      <td>-1.831799</td>\n",
       "      <td>2.218374</td>\n",
       "      <td>-0.224695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.107354</td>\n",
       "      <td>1.155643e+06</td>\n",
       "      <td>24853.5</td>\n",
       "      <td>5.877962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.741706</td>\n",
       "      <td>-1.093711</td>\n",
       "      <td>-340733.611297</td>\n",
       "      <td>2.211239</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>43932.974950</td>\n",
       "      <td>6530.528782</td>\n",
       "      <td>1469.718291</td>\n",
       "      <td>1.949371e+08</td>\n",
       "      <td>5892.504535</td>\n",
       "      <td>0.140442</td>\n",
       "      <td>1218.349279</td>\n",
       "      <td>-1.029972</td>\n",
       "      <td>4.324358</td>\n",
       "      <td>-0.164236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 250 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0             1        2          3         4         5         6    \\\n",
       "0  0.000000  5.518822e+05  20979.0   3.377962 -1.101215 -2.551508 -1.091257   \n",
       "1  0.000000  3.778954e+05  15862.5   0.977962 -1.101215 -2.196168 -1.094868   \n",
       "2  1.107354  2.114595e+06  50544.0  13.377962 -1.101215 -2.523555 -1.093467   \n",
       "3  0.000000  4.317482e+05  13401.0   1.127962 -1.101215 -2.780802 -1.116898   \n",
       "4  1.107354  1.155643e+06  24853.5   5.877962 -1.101215 -2.741706 -1.093711   \n",
       "\n",
       "             7         8        9      ...              240           241  \\\n",
       "0 -107246.452668  1.738143  0.60734    ...     13668.364853    264.722662   \n",
       "1 -570408.734682  2.106473  0.60734    ...     43932.974950   6530.528782   \n",
       "2 -461829.264180  1.616099  0.60734    ...     79464.752034  15362.335289   \n",
       "3 -331457.658872  2.116840  0.60734    ...     65220.968011   9635.096700   \n",
       "4 -340733.611297  2.211239  0.60734    ...     43932.974950   6530.528782   \n",
       "\n",
       "           242           243           244       245          246       247  \\\n",
       "0  1238.200538 -1.265235e+00    296.464297 -0.422659     1.898090 -0.951974   \n",
       "1  1469.718291  1.949371e+08   5892.504535  0.140442  1218.349279 -1.029972   \n",
       "2     3.328419  1.339639e+08  22997.734839 -0.422659     1.898090 -0.734273   \n",
       "3  1868.980047  1.833251e+08   3538.963401 -0.422659     1.898090 -1.831799   \n",
       "4  1469.718291  1.949371e+08   5892.504535  0.140442  1218.349279 -1.029972   \n",
       "\n",
       "        248       249  \n",
       "0  2.875669  1.107406  \n",
       "1  4.324358 -0.164236  \n",
       "2  2.957831 -0.786946  \n",
       "3  2.218374 -0.224695  \n",
       "4  4.324358 -0.164236  \n",
       "\n",
       "[5 rows x 250 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler_rus_train = RobustScaler(with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0), copy=True)\n",
    "scaler_rus_train.fit(train_rus_sel)\n",
    "train_rus_sel_scaled = scaler.transform(train_rus_sel)\n",
    "train_rus_sel_scaled = pd.DataFrame(train_rus_sel_scaled)\n",
    "train_rus_sel_scaled.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:35.415617Z",
     "start_time": "2018-09-07T04:09:34.889172Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>240</th>\n",
       "      <th>241</th>\n",
       "      <th>242</th>\n",
       "      <th>243</th>\n",
       "      <th>244</th>\n",
       "      <th>245</th>\n",
       "      <th>246</th>\n",
       "      <th>247</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.975794e+05</td>\n",
       "      <td>20560.5</td>\n",
       "      <td>3.377962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.856546</td>\n",
       "      <td>-1.094902</td>\n",
       "      <td>-328471.564599</td>\n",
       "      <td>2.240140</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>13964.618541</td>\n",
       "      <td>802.352198</td>\n",
       "      <td>848.862401</td>\n",
       "      <td>1.820318e+06</td>\n",
       "      <td>353.414771</td>\n",
       "      <td>-0.422659</td>\n",
       "      <td>1.898090</td>\n",
       "      <td>-0.798800</td>\n",
       "      <td>3.861613</td>\n",
       "      <td>1.142006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.732032e+05</td>\n",
       "      <td>17370.0</td>\n",
       "      <td>0.377962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.780737</td>\n",
       "      <td>-1.100082</td>\n",
       "      <td>-579303.483582</td>\n",
       "      <td>2.112756</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>14807.182023</td>\n",
       "      <td>814.287232</td>\n",
       "      <td>1034.459861</td>\n",
       "      <td>1.294439e+06</td>\n",
       "      <td>481.887581</td>\n",
       "      <td>-0.422659</td>\n",
       "      <td>1.898090</td>\n",
       "      <td>-1.777019</td>\n",
       "      <td>3.039993</td>\n",
       "      <td>0.445680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.107354</td>\n",
       "      <td>8.134308e+05</td>\n",
       "      <td>69777.0</td>\n",
       "      <td>5.377962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.907879</td>\n",
       "      <td>-1.100056</td>\n",
       "      <td>-138187.472057</td>\n",
       "      <td>1.817464</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>25859.568566</td>\n",
       "      <td>16507.370966</td>\n",
       "      <td>12.898894</td>\n",
       "      <td>7.342543e+07</td>\n",
       "      <td>14500.250260</td>\n",
       "      <td>-2.139490</td>\n",
       "      <td>-30597.451604</td>\n",
       "      <td>-0.672233</td>\n",
       "      <td>4.683233</td>\n",
       "      <td>-2.110397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.931591e+06</td>\n",
       "      <td>49018.5</td>\n",
       "      <td>15.877962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.517436</td>\n",
       "      <td>-1.093782</td>\n",
       "      <td>-127069.035932</td>\n",
       "      <td>1.706729</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>11815.099029</td>\n",
       "      <td>1799.797590</td>\n",
       "      <td>-1.379324</td>\n",
       "      <td>1.725857e+06</td>\n",
       "      <td>4829.839399</td>\n",
       "      <td>-1.345911</td>\n",
       "      <td>-11994.520668</td>\n",
       "      <td>-0.541122</td>\n",
       "      <td>3.532965</td>\n",
       "      <td>-0.103595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.107354</td>\n",
       "      <td>7.671168e+05</td>\n",
       "      <td>32067.0</td>\n",
       "      <td>5.327962</td>\n",
       "      <td>-1.101215</td>\n",
       "      <td>-2.457150</td>\n",
       "      <td>-1.094568</td>\n",
       "      <td>-254136.877362</td>\n",
       "      <td>1.698247</td>\n",
       "      <td>0.60734</td>\n",
       "      <td>...</td>\n",
       "      <td>47555.253804</td>\n",
       "      <td>7154.819990</td>\n",
       "      <td>1581.851162</td>\n",
       "      <td>2.147183e+08</td>\n",
       "      <td>6579.418780</td>\n",
       "      <td>0.425553</td>\n",
       "      <td>2686.418406</td>\n",
       "      <td>-1.055336</td>\n",
       "      <td>3.987481</td>\n",
       "      <td>-0.289294</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 250 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0             1        2          3         4         5         6    \\\n",
       "0  0.000000  6.975794e+05  20560.5   3.377962 -1.101215 -2.856546 -1.094902   \n",
       "1  0.000000  2.732032e+05  17370.0   0.377962 -1.101215 -2.780737 -1.100082   \n",
       "2  1.107354  8.134308e+05  69777.0   5.377962 -1.101215 -2.907879 -1.100056   \n",
       "3  0.000000  1.931591e+06  49018.5  15.877962 -1.101215 -2.517436 -1.093782   \n",
       "4  1.107354  7.671168e+05  32067.0   5.327962 -1.101215 -2.457150 -1.094568   \n",
       "\n",
       "             7         8        9      ...              240           241  \\\n",
       "0 -328471.564599  2.240140  0.60734    ...     13964.618541    802.352198   \n",
       "1 -579303.483582  2.112756  0.60734    ...     14807.182023    814.287232   \n",
       "2 -138187.472057  1.817464  0.60734    ...     25859.568566  16507.370966   \n",
       "3 -127069.035932  1.706729  0.60734    ...     11815.099029   1799.797590   \n",
       "4 -254136.877362  1.698247  0.60734    ...     47555.253804   7154.819990   \n",
       "\n",
       "           242           243           244       245           246       247  \\\n",
       "0   848.862401  1.820318e+06    353.414771 -0.422659      1.898090 -0.798800   \n",
       "1  1034.459861  1.294439e+06    481.887581 -0.422659      1.898090 -1.777019   \n",
       "2    12.898894  7.342543e+07  14500.250260 -2.139490 -30597.451604 -0.672233   \n",
       "3    -1.379324  1.725857e+06   4829.839399 -1.345911 -11994.520668 -0.541122   \n",
       "4  1581.851162  2.147183e+08   6579.418780  0.425553   2686.418406 -1.055336   \n",
       "\n",
       "        248       249  \n",
       "0  3.861613  1.142006  \n",
       "1  3.039993  0.445680  \n",
       "2  4.683233 -2.110397  \n",
       "3  3.532965 -0.103595  \n",
       "4  3.987481 -0.289294  \n",
       "\n",
       "[5 rows x 250 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaler_rus_test = RobustScaler(with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0), copy=True)\n",
    "scaler_rus_test.fit(test_rus_sel)\n",
    "test_rus_sel_scaled = scaler.transform(test_rus_sel)\n",
    "test_rus_sel_scaled = pd.DataFrame(test_rus_sel_scaled)\n",
    "test_rus_sel_scaled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T04:09:35.419424Z",
     "start_time": "2018-09-07T04:09:35.417419Z"
    }
   },
   "outputs": [],
   "source": [
    "# # Saving datasets\n",
    "# np.savetxt('datasets/train_rus_sel_scaled.out', train_rus_sel_scaled, delimiter=',')\n",
    "# np.savetxt('datasets/test_rus_sel_scaled.out', test_rus_sel_scaled, delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:15:07.462099Z",
     "start_time": "2018-09-07T10:15:07.451959Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import lightgbm as lgb\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:15:07.970294Z",
     "start_time": "2018-09-07T10:15:07.956512Z"
    },
    "code_folding": [
     1
    ]
   },
   "outputs": [],
   "source": [
    "# Taken from William Koehrsen and modified.\n",
    "def model(features, test_features, encoding = 'ohe', n_folds = 10, n_estimators=10000, learning_rate = 0.01,\n",
    "         reg_alpha = 0.1, reg_lambda = 0.1):\n",
    "    \n",
    "    \"\"\"Train and test a light gradient boosting model using\n",
    "    cross validation. \n",
    "    \n",
    "    Parameters\n",
    "    --------\n",
    "        features (pd.DataFrame): \n",
    "            dataframe of training features to use \n",
    "            for training a model. Must include the TARGET column.\n",
    "        test_features (pd.DataFrame): \n",
    "            dataframe of testing features to use\n",
    "            for making predictions with the model. \n",
    "        encoding (str, default = 'ohe'): \n",
    "            method for encoding categorical variables. Either 'ohe' for one-hot encoding or 'le' for integer label encoding\n",
    "            n_folds (int, default = 5): number of folds to use for cross validation\n",
    "        \n",
    "    Return\n",
    "    --------\n",
    "        submission (pd.DataFrame): \n",
    "            dataframe with `SK_ID_CURR` and `TARGET` probabilities\n",
    "            predicted by the model.\n",
    "        feature_importances (pd.DataFrame): \n",
    "            dataframe with the feature importances from the model.\n",
    "        valid_metrics (pd.DataFrame): \n",
    "            dataframe with training and validation metrics (ROC AUC) for each fold and overall.\n",
    "        \n",
    "    \"\"\"\n",
    "    \n",
    "    # Extract the ids\n",
    "    if 'SK_ID_CURR' in features.columns:\n",
    "        train_ids = features['SK_ID_CURR']\n",
    "    \n",
    "    if 'SK_ID_CURR' in test_features.columns:\n",
    "        test_ids = test_features['SK_ID_CURR']\n",
    "    \n",
    "    # Extract the labels for training\n",
    "    labels = features['TARGET']\n",
    "    \n",
    "    # Remove the ids and target\n",
    "    features = features.drop(columns = ['TARGET'])\n",
    "    if 'SK_ID_CURR' in features.columns:\n",
    "        features = features.drop(columns = ['SK_ID_CURR'])\n",
    "    \n",
    "    if 'SK_ID_CURR' in test_features.columns:\n",
    "        test_features = test_features.drop(columns = ['SK_ID_CURR'])\n",
    "    \n",
    "    \n",
    "    # One Hot Encoding\n",
    "    if encoding == 'ohe':\n",
    "        features = pd.get_dummies(features)\n",
    "        test_features = pd.get_dummies(test_features)\n",
    "        \n",
    "        # Align the dataframes by the columns\n",
    "        features, test_features = features.align(test_features, join = 'inner', axis = 1)\n",
    "        \n",
    "        # No categorical indices to record\n",
    "        cat_indices = 'auto'\n",
    "    \n",
    "    # Integer label encoding\n",
    "    elif encoding == 'le':\n",
    "        \n",
    "        # Create a label encoder\n",
    "        label_encoder = LabelEncoder()\n",
    "        \n",
    "        # List for storing categorical indices\n",
    "        cat_indices = []\n",
    "        \n",
    "        # Iterate through each column\n",
    "        for i, col in enumerate(features):\n",
    "            if features[col].dtype == 'object':\n",
    "                # Map the categorical features to integers\n",
    "                features[col] = label_encoder.fit_transform(np.array(features[col].astype(str)).reshape((-1,)))\n",
    "                test_features[col] = label_encoder.transform(np.array(test_features[col].astype(str)).reshape((-1,)))\n",
    "\n",
    "                # Record the categorical indices\n",
    "                cat_indices.append(i)\n",
    "    \n",
    "    # Catch error if label encoding scheme is not valid\n",
    "    else:\n",
    "        raise ValueError(\"Encoding must be either 'ohe' or 'le'\")\n",
    "        \n",
    "    print('Training Data Shape: ', features.shape)\n",
    "    print('Testing Data Shape: ', test_features.shape)\n",
    "    \n",
    "    # Extract feature names\n",
    "    feature_names = list(features.columns)\n",
    "    \n",
    "    # Convert to np arrays\n",
    "    features = np.array(features)\n",
    "    test_features = np.array(test_features)\n",
    "    \n",
    "    # Create the kfold object\n",
    "    k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 53)\n",
    "    \n",
    "    # Empty array for feature importances\n",
    "    feature_importance_values = np.zeros(len(feature_names))\n",
    "    \n",
    "    # Empty array for test predictions\n",
    "    test_predictions = np.zeros(test_features.shape[0])\n",
    "    \n",
    "    # Empty array for out of fold validation predictions\n",
    "    out_of_fold = np.zeros(features.shape[0])\n",
    "    \n",
    "    # Lists for recording validation and training scores\n",
    "    valid_scores = []\n",
    "    train_scores = []\n",
    "    \n",
    "    # Iterate through each fold\n",
    "    for train_indices, valid_indices in k_fold.split(features):\n",
    "        \n",
    "        # Training data for the fold\n",
    "        train_features, train_labels = features[train_indices], labels[train_indices]\n",
    "        # Validation data for the fold\n",
    "        valid_features, valid_labels = features[valid_indices], labels[valid_indices]\n",
    "        \n",
    "        # Create the model\n",
    "        model = lgb.LGBMClassifier(n_estimators=n_estimators, objective = 'binary', \n",
    "                                   class_weight = 'balanced', learning_rate = learning_rate, \n",
    "                                   reg_alpha = reg_alpha, reg_lambda = reg_lambda, \n",
    "                                   subsample = 0.8, n_jobs = -1, random_state = 50)\n",
    "        \n",
    "        # Train the model\n",
    "        model.fit(train_features, train_labels, eval_metric = 'auc',\n",
    "                  eval_set = [(valid_features, valid_labels), (train_features, train_labels)],\n",
    "                  eval_names = ['valid', 'train'], categorical_feature = cat_indices,\n",
    "                  early_stopping_rounds = 100, verbose = 200)\n",
    "        \n",
    "        # Record the best iteration\n",
    "        best_iteration = model.best_iteration_\n",
    "        \n",
    "        # Record the feature importances\n",
    "        feature_importance_values += model.feature_importances_ / k_fold.n_splits\n",
    "        \n",
    "        # Make predictions\n",
    "        test_predictions += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits\n",
    "        \n",
    "        # Record the out of fold predictions\n",
    "        out_of_fold[valid_indices] = model.predict_proba(valid_features, num_iteration = best_iteration)[:, 1]\n",
    "        \n",
    "        # Record the best score\n",
    "        valid_score = model.best_score_['valid']['auc']\n",
    "        train_score = model.best_score_['train']['auc']\n",
    "        \n",
    "        valid_scores.append(valid_score)\n",
    "        train_scores.append(train_score)\n",
    "        \n",
    "        # Clean up memory\n",
    "        gc.enable()\n",
    "        del model, train_features, valid_features\n",
    "        gc.collect()\n",
    "        \n",
    "    # Make the submission dataframe\n",
    "    submission = pd.DataFrame({'SK_ID_CURR': test_id, 'TARGET': test_predictions})\n",
    "    \n",
    "    # Make the feature importance dataframe\n",
    "    feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})\n",
    "    \n",
    "    # Overall validation score\n",
    "    valid_auc = roc_auc_score(labels, out_of_fold)\n",
    "    \n",
    "    # Add the overall scores to the metrics\n",
    "    valid_scores.append(valid_auc)\n",
    "    train_scores.append(np.mean(train_scores))\n",
    "    \n",
    "    # Needed for creating dataframe of validation scores\n",
    "    fold_names = list(range(n_folds))\n",
    "    fold_names.append('overall')\n",
    "    \n",
    "    # Dataframe of validation scores\n",
    "    metrics = pd.DataFrame({'fold': fold_names,\n",
    "                            'train': train_scores,\n",
    "                            'valid': valid_scores}) \n",
    "    \n",
    "    return submission, feature_importances, metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:19:04.948871Z",
     "start_time": "2018-09-07T10:15:08.464293Z"
    }
   },
   "outputs": [],
   "source": [
    "train_os_sel = np.loadtxt('datasets/train_os_sel.out', delimiter=',')\n",
    "train_rus_sel = np.loadtxt('datasets/train_rus_sel.out', delimiter=',')\n",
    "train_os_pca = np.loadtxt('datasets/train_os_pca.out', delimiter=',')\n",
    "train_rus_pca = np.loadtxt('datasets/train_rus_pca.out', delimiter=',')\n",
    "train_os_sel_scaled = np.loadtxt('datasets/train_os_sel_scaled.out', delimiter=',')\n",
    "train_rus_sel_scaled = np.loadtxt('datasets/train_rus_sel_scaled.out', delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:19:42.597461Z",
     "start_time": "2018-09-07T10:19:05.499143Z"
    }
   },
   "outputs": [],
   "source": [
    "test_os_sel = np.loadtxt('datasets/test_os_sel.out', delimiter=',')\n",
    "test_rus_sel = np.loadtxt('datasets/test_rus_sel.out', delimiter=',')\n",
    "test_os_pca = np.loadtxt('datasets/test_os_pca.out', delimiter=',')\n",
    "test_rus_pca = np.loadtxt('datasets/test_rus_pca.out', delimiter=',')\n",
    "test_os_sel_scaled = np.loadtxt('datasets/test_os_sel_scaled.out', delimiter=',')\n",
    "test_rus_sel_scaled = np.loadtxt('datasets/test_rus_sel_scaled.out', delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:22:51.206070Z",
     "start_time": "2018-09-07T10:22:51.190024Z"
    }
   },
   "outputs": [],
   "source": [
    "train['TARGET'] = train_labels\n",
    "\n",
    "train_imp['TARGET'] = train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:23:29.125388Z",
     "start_time": "2018-09-07T10:23:29.097488Z"
    }
   },
   "outputs": [],
   "source": [
    "train_os = pd.DataFrame(train_os)\n",
    "train_os.columns = train_imp.columns.drop('TARGET')\n",
    "train_os['TARGET'] = label_os\n",
    "\n",
    "train_rus = pd.DataFrame(train_rus)\n",
    "train_rus.columns = train_imp.columns.drop('TARGET')\n",
    "train_rus['TARGET'] = label_rus\n",
    "\n",
    "train_os_sel = pd.DataFrame(train_os_sel)\n",
    "train_os_sel['TARGET'] = label_os\n",
    "\n",
    "train_rus_sel = pd.DataFrame(train_rus_sel)\n",
    "train_rus_sel['TARGET'] = label_rus\n",
    "\n",
    "train_os_pca = pd.DataFrame(train_os_pca)\n",
    "train_os_pca['TARGET'] = label_os\n",
    "\n",
    "train_rus_pca = pd.DataFrame(train_rus_pca)\n",
    "train_rus_pca['TARGET'] = label_rus\n",
    "\n",
    "train_os_sel_scaled = pd.DataFrame(train_os_sel_scaled)\n",
    "train_os_sel_scaled['TARGET'] = label_os\n",
    "\n",
    "train_rus_sel_scaled = pd.DataFrame(train_rus_sel_scaled)\n",
    "train_rus_sel_scaled['TARGET'] = label_rus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T11:07:58.898734Z",
     "start_time": "2018-09-07T11:07:58.891112Z"
    }
   },
   "outputs": [],
   "source": [
    "test_os_sel = pd.DataFrame(test_os_sel)\n",
    "test_rus_sel = pd.DataFrame(test_rus_sel)\n",
    "test_os_pca = pd.DataFrame(test_os_pca)\n",
    "test_rus_pca = pd.DataFrame(test_rus_pca)\n",
    "test_os_sel_scaled = pd.DataFrame(test_os_sel_scaled)\n",
    "test_rus_sel_scaled = pd.DataFrame(test_rus_sel_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:23:37.488315Z",
     "start_time": "2018-09-07T10:23:37.483598Z"
    }
   },
   "outputs": [],
   "source": [
    "trainsets = [train, train_imp, train_os, train_rus, train_os_sel, train_rus_sel, train_os_pca, train_rus_pca,\n",
    "            train_os_sel_scaled, train_rus_sel_scaled]\n",
    "testsets = [test, test_imp, test_imp, test_imp, test_os_sel, test_rus_sel, test_os_pca, test_rus_pca,\n",
    "            test_os_sel_scaled, test_rus_sel_scaled]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T10:23:50.419838Z",
     "start_time": "2018-09-07T10:23:50.415182Z"
    }
   },
   "outputs": [],
   "source": [
    "result_names = ['raw', 'imputed', 'oversampled', 'undersampled', 'oversampled_selected', 'undersampled_selected', \n",
    "                'oversampled_pca', 'undersampled_pca', 'oversampled_selected_scaled', 'undersampled_selected_scaled']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-07T12:24:27.223199Z",
     "start_time": "2018-09-07T11:09:39.590052Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Shape:  (565372, 250)\n",
      "Testing Data Shape:  (48744, 250)\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.979511\ttrain's auc: 0.980071\n",
      "[400]\tvalid's auc: 0.97998\ttrain's auc: 0.98437\n",
      "Early stopping, best iteration is:\n",
      "[469]\tvalid's auc: 0.980025\ttrain's auc: 0.985569\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977087\ttrain's auc: 0.98032\n",
      "[400]\tvalid's auc: 0.977489\ttrain's auc: 0.984451\n",
      "Early stopping, best iteration is:\n",
      "[488]\tvalid's auc: 0.97753\ttrain's auc: 0.985943\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.978061\ttrain's auc: 0.980188\n",
      "[400]\tvalid's auc: 0.978589\ttrain's auc: 0.984459\n",
      "[600]\tvalid's auc: 0.978662\ttrain's auc: 0.987592\n",
      "Early stopping, best iteration is:\n",
      "[639]\tvalid's auc: 0.9787\ttrain's auc: 0.988108\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.978594\ttrain's auc: 0.980237\n",
      "[400]\tvalid's auc: 0.97906\ttrain's auc: 0.984338\n",
      "[600]\tvalid's auc: 0.979049\ttrain's auc: 0.987479\n",
      "Early stopping, best iteration is:\n",
      "[529]\tvalid's auc: 0.979095\ttrain's auc: 0.986484\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977837\ttrain's auc: 0.980221\n",
      "[400]\tvalid's auc: 0.978263\ttrain's auc: 0.984445\n",
      "Early stopping, best iteration is:\n",
      "[445]\tvalid's auc: 0.978308\ttrain's auc: 0.985186\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977933\ttrain's auc: 0.980235\n",
      "[400]\tvalid's auc: 0.978356\ttrain's auc: 0.984458\n",
      "Early stopping, best iteration is:\n",
      "[374]\tvalid's auc: 0.978382\ttrain's auc: 0.983947\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.978296\ttrain's auc: 0.980192\n",
      "[400]\tvalid's auc: 0.978832\ttrain's auc: 0.984391\n",
      "[600]\tvalid's auc: 0.978885\ttrain's auc: 0.987527\n",
      "Early stopping, best iteration is:\n",
      "[683]\tvalid's auc: 0.97892\ttrain's auc: 0.988556\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977764\ttrain's auc: 0.980279\n",
      "[400]\tvalid's auc: 0.978007\ttrain's auc: 0.984469\n",
      "Early stopping, best iteration is:\n",
      "[334]\tvalid's auc: 0.978047\ttrain's auc: 0.983172\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.978339\ttrain's auc: 0.980176\n",
      "[400]\tvalid's auc: 0.978621\ttrain's auc: 0.984354\n",
      "[600]\tvalid's auc: 0.97866\ttrain's auc: 0.987505\n",
      "Early stopping, best iteration is:\n",
      "[502]\tvalid's auc: 0.978671\ttrain's auc: 0.986121\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.97784\ttrain's auc: 0.980307\n",
      "[400]\tvalid's auc: 0.978415\ttrain's auc: 0.984522\n",
      "[600]\tvalid's auc: 0.978565\ttrain's auc: 0.987706\n",
      "Early stopping, best iteration is:\n",
      "[596]\tvalid's auc: 0.978572\ttrain's auc: 0.987659\n",
      "Training Data Shape:  (49650, 250)\n",
      "Testing Data Shape:  (48744, 250)\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.759697\ttrain's auc: 0.842829\n",
      "[400]\tvalid's auc: 0.760693\ttrain's auc: 0.889912\n",
      "Early stopping, best iteration is:\n",
      "[362]\tvalid's auc: 0.760927\ttrain's auc: 0.882089\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.780166\ttrain's auc: 0.841258\n",
      "[400]\tvalid's auc: 0.780845\ttrain's auc: 0.889196\n",
      "Early stopping, best iteration is:\n",
      "[423]\tvalid's auc: 0.781341\ttrain's auc: 0.893875\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.775308\ttrain's auc: 0.84111\n",
      "Early stopping, best iteration is:\n",
      "[272]\tvalid's auc: 0.776286\ttrain's auc: 0.860396\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.770815\ttrain's auc: 0.841636\n",
      "[400]\tvalid's auc: 0.772923\ttrain's auc: 0.890173\n",
      "[600]\tvalid's auc: 0.773267\ttrain's auc: 0.924044\n",
      "Early stopping, best iteration is:\n",
      "[607]\tvalid's auc: 0.773508\ttrain's auc: 0.924993\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.776122\ttrain's auc: 0.841254\n",
      "Early stopping, best iteration is:\n",
      "[209]\tvalid's auc: 0.776548\ttrain's auc: 0.843773\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.782298\ttrain's auc: 0.840727\n",
      "[400]\tvalid's auc: 0.783468\ttrain's auc: 0.888963\n",
      "Early stopping, best iteration is:\n",
      "[305]\tvalid's auc: 0.784163\ttrain's auc: 0.868186\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.772001\ttrain's auc: 0.841069\n",
      "[400]\tvalid's auc: 0.773264\ttrain's auc: 0.889582\n",
      "Early stopping, best iteration is:\n",
      "[317]\tvalid's auc: 0.773844\ttrain's auc: 0.871233\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.766533\ttrain's auc: 0.841334\n",
      "[400]\tvalid's auc: 0.768123\ttrain's auc: 0.888765\n",
      "[600]\tvalid's auc: 0.768617\ttrain's auc: 0.922624\n",
      "Early stopping, best iteration is:\n",
      "[601]\tvalid's auc: 0.768668\ttrain's auc: 0.92278\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.774017\ttrain's auc: 0.84101\n",
      "Early stopping, best iteration is:\n",
      "[196]\tvalid's auc: 0.774184\ttrain's auc: 0.839775\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.762077\ttrain's auc: 0.842102\n",
      "[400]\tvalid's auc: 0.762796\ttrain's auc: 0.890448\n",
      "Early stopping, best iteration is:\n",
      "[351]\tvalid's auc: 0.763252\ttrain's auc: 0.880045\n",
      "Training Data Shape:  (565372, 50)\n",
      "Testing Data Shape:  (48744, 50)\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.881694\ttrain's auc: 0.885725\n",
      "[400]\tvalid's auc: 0.904227\ttrain's auc: 0.909675\n",
      "[600]\tvalid's auc: 0.912908\ttrain's auc: 0.920697\n",
      "[800]\tvalid's auc: 0.919003\ttrain's auc: 0.928783\n",
      "[1000]\tvalid's auc: 0.924032\ttrain's auc: 0.93544\n",
      "[1200]\tvalid's auc: 0.928476\ttrain's auc: 0.94121\n",
      "[1400]\tvalid's auc: 0.932243\ttrain's auc: 0.946173\n",
      "[1600]\tvalid's auc: 0.935674\ttrain's auc: 0.950647\n",
      "[1800]\tvalid's auc: 0.938552\ttrain's auc: 0.954415\n",
      "[2000]\tvalid's auc: 0.941237\ttrain's auc: 0.958043\n",
      "[2200]\tvalid's auc: 0.943716\ttrain's auc: 0.961208\n",
      "[2400]\tvalid's auc: 0.945791\ttrain's auc: 0.963987\n",
      "[2600]\tvalid's auc: 0.947799\ttrain's auc: 0.966506\n",
      "[2800]\tvalid's auc: 0.949635\ttrain's auc: 0.968834\n",
      "[3000]\tvalid's auc: 0.951459\ttrain's auc: 0.970985\n",
      "[3200]\tvalid's auc: 0.95322\ttrain's auc: 0.973156\n",
      "[3400]\tvalid's auc: 0.954759\ttrain's auc: 0.975024\n",
      "[3600]\tvalid's auc: 0.9562\ttrain's auc: 0.97672\n",
      "[3800]\tvalid's auc: 0.957593\ttrain's auc: 0.978292\n",
      "[4000]\tvalid's auc: 0.958884\ttrain's auc: 0.979775\n",
      "[4200]\tvalid's auc: 0.960155\ttrain's auc: 0.981239\n",
      "[4400]\tvalid's auc: 0.961321\ttrain's auc: 0.982512\n",
      "[4600]\tvalid's auc: 0.962464\ttrain's auc: 0.98373\n",
      "[4800]\tvalid's auc: 0.963518\ttrain's auc: 0.98478\n",
      "[5000]\tvalid's auc: 0.964523\ttrain's auc: 0.985777\n",
      "[5200]\tvalid's auc: 0.965439\ttrain's auc: 0.986702\n",
      "[5400]\tvalid's auc: 0.966317\ttrain's auc: 0.987524\n",
      "[5600]\tvalid's auc: 0.967168\ttrain's auc: 0.988265\n",
      "[5800]\tvalid's auc: 0.967957\ttrain's auc: 0.989016\n",
      "[6000]\tvalid's auc: 0.968798\ttrain's auc: 0.98976\n",
      "[6200]\tvalid's auc: 0.969539\ttrain's auc: 0.990409\n",
      "[6400]\tvalid's auc: 0.970267\ttrain's auc: 0.991057\n",
      "[6600]\tvalid's auc: 0.970948\ttrain's auc: 0.99162\n",
      "[6800]\tvalid's auc: 0.971528\ttrain's auc: 0.992112\n",
      "[7000]\tvalid's auc: 0.972209\ttrain's auc: 0.99266\n",
      "[7200]\tvalid's auc: 0.9728\ttrain's auc: 0.993131\n",
      "[7400]\tvalid's auc: 0.973368\ttrain's auc: 0.993581\n",
      "[7600]\tvalid's auc: 0.973903\ttrain's auc: 0.993954\n",
      "[7800]\tvalid's auc: 0.974477\ttrain's auc: 0.994363\n",
      "[8000]\tvalid's auc: 0.974953\ttrain's auc: 0.994738\n",
      "[8200]\tvalid's auc: 0.975427\ttrain's auc: 0.995071\n",
      "[8400]\tvalid's auc: 0.975851\ttrain's auc: 0.995366\n",
      "[8600]\tvalid's auc: 0.976235\ttrain's auc: 0.995639\n",
      "[8800]\tvalid's auc: 0.976661\ttrain's auc: 0.995928\n",
      "[9000]\tvalid's auc: 0.977087\ttrain's auc: 0.996199\n",
      "[9200]\tvalid's auc: 0.977434\ttrain's auc: 0.996414\n",
      "[9400]\tvalid's auc: 0.977814\ttrain's auc: 0.996634\n",
      "[9600]\tvalid's auc: 0.978195\ttrain's auc: 0.996865\n",
      "[9800]\tvalid's auc: 0.978529\ttrain's auc: 0.997063\n",
      "[10000]\tvalid's auc: 0.978866\ttrain's auc: 0.997225\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.978866\ttrain's auc: 0.997225\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.879401\ttrain's auc: 0.886454\n",
      "[400]\tvalid's auc: 0.901425\ttrain's auc: 0.90986\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[600]\tvalid's auc: 0.911004\ttrain's auc: 0.921319\n",
      "[800]\tvalid's auc: 0.917225\ttrain's auc: 0.92936\n",
      "[1000]\tvalid's auc: 0.922553\ttrain's auc: 0.936159\n",
      "[1200]\tvalid's auc: 0.926889\ttrain's auc: 0.941672\n",
      "[1400]\tvalid's auc: 0.930861\ttrain's auc: 0.94673\n",
      "[1600]\tvalid's auc: 0.934233\ttrain's auc: 0.95109\n",
      "[1800]\tvalid's auc: 0.937183\ttrain's auc: 0.95501\n",
      "[2000]\tvalid's auc: 0.939827\ttrain's auc: 0.958366\n",
      "[2200]\tvalid's auc: 0.942365\ttrain's auc: 0.961545\n",
      "[2400]\tvalid's auc: 0.944737\ttrain's auc: 0.964474\n",
      "[2600]\tvalid's auc: 0.94693\ttrain's auc: 0.967172\n",
      "[2800]\tvalid's auc: 0.948937\ttrain's auc: 0.969585\n",
      "[3000]\tvalid's auc: 0.950673\ttrain's auc: 0.971767\n",
      "[3200]\tvalid's auc: 0.952274\ttrain's auc: 0.973669\n",
      "[3400]\tvalid's auc: 0.953765\ttrain's auc: 0.975359\n",
      "[3600]\tvalid's auc: 0.955276\ttrain's auc: 0.976975\n",
      "[3800]\tvalid's auc: 0.956659\ttrain's auc: 0.978574\n",
      "[4000]\tvalid's auc: 0.957935\ttrain's auc: 0.979967\n",
      "[4200]\tvalid's auc: 0.959265\ttrain's auc: 0.981427\n",
      "[4400]\tvalid's auc: 0.960464\ttrain's auc: 0.982676\n",
      "[4600]\tvalid's auc: 0.961624\ttrain's auc: 0.983854\n",
      "[4800]\tvalid's auc: 0.962561\ttrain's auc: 0.984853\n",
      "[5000]\tvalid's auc: 0.963611\ttrain's auc: 0.985848\n",
      "[5200]\tvalid's auc: 0.964509\ttrain's auc: 0.986721\n",
      "[5400]\tvalid's auc: 0.965393\ttrain's auc: 0.987563\n",
      "[5600]\tvalid's auc: 0.966249\ttrain's auc: 0.988302\n",
      "[5800]\tvalid's auc: 0.967028\ttrain's auc: 0.988981\n",
      "[6000]\tvalid's auc: 0.967768\ttrain's auc: 0.989672\n",
      "[6200]\tvalid's auc: 0.968531\ttrain's auc: 0.990336\n",
      "[6400]\tvalid's auc: 0.969232\ttrain's auc: 0.990957\n",
      "[6600]\tvalid's auc: 0.96995\ttrain's auc: 0.991575\n",
      "[6800]\tvalid's auc: 0.970662\ttrain's auc: 0.99217\n",
      "[7000]\tvalid's auc: 0.971269\ttrain's auc: 0.99266\n",
      "[7200]\tvalid's auc: 0.971925\ttrain's auc: 0.993175\n",
      "[7400]\tvalid's auc: 0.972522\ttrain's auc: 0.993611\n",
      "[7600]\tvalid's auc: 0.973106\ttrain's auc: 0.994055\n",
      "[7800]\tvalid's auc: 0.973672\ttrain's auc: 0.994436\n",
      "[8000]\tvalid's auc: 0.974158\ttrain's auc: 0.994809\n",
      "[8200]\tvalid's auc: 0.974635\ttrain's auc: 0.995162\n",
      "[8400]\tvalid's auc: 0.975127\ttrain's auc: 0.995488\n",
      "[8600]\tvalid's auc: 0.975575\ttrain's auc: 0.995783\n",
      "[8800]\tvalid's auc: 0.976064\ttrain's auc: 0.996061\n",
      "[9000]\tvalid's auc: 0.976494\ttrain's auc: 0.996295\n",
      "[9200]\tvalid's auc: 0.976876\ttrain's auc: 0.996516\n",
      "[9400]\tvalid's auc: 0.977279\ttrain's auc: 0.996746\n",
      "[9600]\tvalid's auc: 0.977656\ttrain's auc: 0.996947\n",
      "[9800]\tvalid's auc: 0.97802\ttrain's auc: 0.997147\n",
      "[10000]\tvalid's auc: 0.978393\ttrain's auc: 0.997328\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.978393\ttrain's auc: 0.997328\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.88045\ttrain's auc: 0.886408\n",
      "[400]\tvalid's auc: 0.902555\ttrain's auc: 0.909925\n",
      "[600]\tvalid's auc: 0.911362\ttrain's auc: 0.920959\n",
      "[800]\tvalid's auc: 0.917567\ttrain's auc: 0.92913\n",
      "[1000]\tvalid's auc: 0.922814\ttrain's auc: 0.935875\n",
      "[1200]\tvalid's auc: 0.927173\ttrain's auc: 0.941491\n",
      "[1400]\tvalid's auc: 0.93122\ttrain's auc: 0.946695\n",
      "[1600]\tvalid's auc: 0.934605\ttrain's auc: 0.950891\n",
      "[1800]\tvalid's auc: 0.937538\ttrain's auc: 0.954708\n",
      "[2000]\tvalid's auc: 0.940175\ttrain's auc: 0.958036\n",
      "[2200]\tvalid's auc: 0.942676\ttrain's auc: 0.961258\n",
      "[2400]\tvalid's auc: 0.944941\ttrain's auc: 0.964135\n",
      "[2600]\tvalid's auc: 0.947085\ttrain's auc: 0.966934\n",
      "[2800]\tvalid's auc: 0.949082\ttrain's auc: 0.96935\n",
      "[3000]\tvalid's auc: 0.950814\ttrain's auc: 0.9715\n",
      "[3200]\tvalid's auc: 0.952578\ttrain's auc: 0.973521\n",
      "[3400]\tvalid's auc: 0.954093\ttrain's auc: 0.975364\n",
      "[3600]\tvalid's auc: 0.955454\ttrain's auc: 0.97699\n",
      "[3800]\tvalid's auc: 0.956909\ttrain's auc: 0.978564\n",
      "[4000]\tvalid's auc: 0.958342\ttrain's auc: 0.98011\n",
      "[4200]\tvalid's auc: 0.959541\ttrain's auc: 0.981413\n",
      "[4400]\tvalid's auc: 0.960734\ttrain's auc: 0.982631\n",
      "[4600]\tvalid's auc: 0.961784\ttrain's auc: 0.983786\n",
      "[4800]\tvalid's auc: 0.962871\ttrain's auc: 0.984901\n",
      "[5000]\tvalid's auc: 0.963868\ttrain's auc: 0.985921\n",
      "[5200]\tvalid's auc: 0.964766\ttrain's auc: 0.986863\n",
      "[5400]\tvalid's auc: 0.965709\ttrain's auc: 0.987715\n",
      "[5600]\tvalid's auc: 0.966574\ttrain's auc: 0.988533\n",
      "[5800]\tvalid's auc: 0.967389\ttrain's auc: 0.989253\n",
      "[6000]\tvalid's auc: 0.968158\ttrain's auc: 0.989985\n",
      "[6200]\tvalid's auc: 0.968842\ttrain's auc: 0.990599\n",
      "[6400]\tvalid's auc: 0.969492\ttrain's auc: 0.991186\n",
      "[6600]\tvalid's auc: 0.970217\ttrain's auc: 0.991771\n",
      "[6800]\tvalid's auc: 0.970901\ttrain's auc: 0.992297\n",
      "[7000]\tvalid's auc: 0.971489\ttrain's auc: 0.992779\n",
      "[7200]\tvalid's auc: 0.972056\ttrain's auc: 0.993213\n",
      "[7400]\tvalid's auc: 0.972566\ttrain's auc: 0.993603\n",
      "[7600]\tvalid's auc: 0.973189\ttrain's auc: 0.994074\n",
      "[7800]\tvalid's auc: 0.973724\ttrain's auc: 0.994438\n",
      "[8000]\tvalid's auc: 0.97421\ttrain's auc: 0.994773\n",
      "[8200]\tvalid's auc: 0.974678\ttrain's auc: 0.995076\n",
      "[8400]\tvalid's auc: 0.975172\ttrain's auc: 0.995411\n",
      "[8600]\tvalid's auc: 0.975621\ttrain's auc: 0.995702\n",
      "[8800]\tvalid's auc: 0.976055\ttrain's auc: 0.995955\n",
      "[9000]\tvalid's auc: 0.97651\ttrain's auc: 0.99623\n",
      "[9200]\tvalid's auc: 0.976939\ttrain's auc: 0.996471\n",
      "[9400]\tvalid's auc: 0.977363\ttrain's auc: 0.996712\n",
      "[9600]\tvalid's auc: 0.977782\ttrain's auc: 0.996935\n",
      "[9800]\tvalid's auc: 0.978128\ttrain's auc: 0.99713\n",
      "[10000]\tvalid's auc: 0.978464\ttrain's auc: 0.997296\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.978464\ttrain's auc: 0.997296\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.882311\ttrain's auc: 0.884784\n",
      "[400]\tvalid's auc: 0.905254\ttrain's auc: 0.909062\n",
      "[600]\tvalid's auc: 0.914564\ttrain's auc: 0.920574\n",
      "[800]\tvalid's auc: 0.920902\ttrain's auc: 0.928961\n",
      "[1000]\tvalid's auc: 0.925652\ttrain's auc: 0.935554\n",
      "[1200]\tvalid's auc: 0.929761\ttrain's auc: 0.941162\n",
      "[1400]\tvalid's auc: 0.933352\ttrain's auc: 0.946073\n",
      "[1600]\tvalid's auc: 0.936803\ttrain's auc: 0.950607\n",
      "[1800]\tvalid's auc: 0.939701\ttrain's auc: 0.95436\n",
      "[2000]\tvalid's auc: 0.942295\ttrain's auc: 0.957649\n",
      "[2200]\tvalid's auc: 0.944817\ttrain's auc: 0.960941\n",
      "[2400]\tvalid's auc: 0.947127\ttrain's auc: 0.963865\n",
      "[2600]\tvalid's auc: 0.949143\ttrain's auc: 0.966518\n",
      "[2800]\tvalid's auc: 0.951101\ttrain's auc: 0.968973\n",
      "[3000]\tvalid's auc: 0.952804\ttrain's auc: 0.971094\n",
      "[3200]\tvalid's auc: 0.954384\ttrain's auc: 0.973072\n",
      "[3400]\tvalid's auc: 0.955962\ttrain's auc: 0.975033\n",
      "[3600]\tvalid's auc: 0.95744\ttrain's auc: 0.976796\n",
      "[3800]\tvalid's auc: 0.95878\ttrain's auc: 0.978403\n",
      "[4000]\tvalid's auc: 0.960107\ttrain's auc: 0.97992\n",
      "[4200]\tvalid's auc: 0.961264\ttrain's auc: 0.981187\n",
      "[4400]\tvalid's auc: 0.962317\ttrain's auc: 0.982436\n",
      "[4600]\tvalid's auc: 0.963365\ttrain's auc: 0.983561\n",
      "[4800]\tvalid's auc: 0.964294\ttrain's auc: 0.984602\n",
      "[5000]\tvalid's auc: 0.965192\ttrain's auc: 0.985544\n",
      "[5200]\tvalid's auc: 0.96603\ttrain's auc: 0.986393\n",
      "[5400]\tvalid's auc: 0.966889\ttrain's auc: 0.987262\n",
      "[5600]\tvalid's auc: 0.967721\ttrain's auc: 0.988076\n",
      "[5800]\tvalid's auc: 0.968453\ttrain's auc: 0.988812\n",
      "[6000]\tvalid's auc: 0.969221\ttrain's auc: 0.989512\n",
      "[6200]\tvalid's auc: 0.969924\ttrain's auc: 0.990179\n",
      "[6400]\tvalid's auc: 0.970607\ttrain's auc: 0.990771\n",
      "[6600]\tvalid's auc: 0.971315\ttrain's auc: 0.991409\n",
      "[6800]\tvalid's auc: 0.971911\ttrain's auc: 0.991918\n",
      "[7000]\tvalid's auc: 0.972531\ttrain's auc: 0.992472\n",
      "[7200]\tvalid's auc: 0.973123\ttrain's auc: 0.992962\n",
      "[7400]\tvalid's auc: 0.973691\ttrain's auc: 0.993438\n",
      "[7600]\tvalid's auc: 0.974218\ttrain's auc: 0.993829\n",
      "[7800]\tvalid's auc: 0.974762\ttrain's auc: 0.994268\n",
      "[8000]\tvalid's auc: 0.975216\ttrain's auc: 0.994624\n",
      "[8200]\tvalid's auc: 0.975721\ttrain's auc: 0.994985\n",
      "[8400]\tvalid's auc: 0.976171\ttrain's auc: 0.995317\n",
      "[8600]\tvalid's auc: 0.976684\ttrain's auc: 0.995699\n",
      "[8800]\tvalid's auc: 0.977094\ttrain's auc: 0.995994\n",
      "[9000]\tvalid's auc: 0.977542\ttrain's auc: 0.996271\n",
      "[9200]\tvalid's auc: 0.97797\ttrain's auc: 0.996548\n",
      "[9400]\tvalid's auc: 0.978334\ttrain's auc: 0.996769\n",
      "[9600]\tvalid's auc: 0.978663\ttrain's auc: 0.996962\n",
      "[9800]\tvalid's auc: 0.97905\ttrain's auc: 0.997203\n",
      "[10000]\tvalid's auc: 0.979415\ttrain's auc: 0.997396\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.979415\ttrain's auc: 0.997396\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.880472\ttrain's auc: 0.886474\n",
      "[400]\tvalid's auc: 0.902348\ttrain's auc: 0.909563\n",
      "[600]\tvalid's auc: 0.911431\ttrain's auc: 0.92089\n",
      "[800]\tvalid's auc: 0.917797\ttrain's auc: 0.929082\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1000]\tvalid's auc: 0.922744\ttrain's auc: 0.935544\n",
      "[1200]\tvalid's auc: 0.927239\ttrain's auc: 0.941281\n",
      "[1400]\tvalid's auc: 0.931103\ttrain's auc: 0.946209\n",
      "[1600]\tvalid's auc: 0.93429\ttrain's auc: 0.950475\n",
      "[1800]\tvalid's auc: 0.937314\ttrain's auc: 0.954398\n",
      "[2000]\tvalid's auc: 0.940019\ttrain's auc: 0.957823\n",
      "[2200]\tvalid's auc: 0.942522\ttrain's auc: 0.961003\n",
      "[2400]\tvalid's auc: 0.944692\ttrain's auc: 0.963837\n",
      "[2600]\tvalid's auc: 0.9469\ttrain's auc: 0.966469\n",
      "[2800]\tvalid's auc: 0.948833\ttrain's auc: 0.968888\n",
      "[3000]\tvalid's auc: 0.950618\ttrain's auc: 0.971071\n",
      "[3200]\tvalid's auc: 0.952356\ttrain's auc: 0.97317\n",
      "[3400]\tvalid's auc: 0.953958\ttrain's auc: 0.975008\n",
      "[3600]\tvalid's auc: 0.955436\ttrain's auc: 0.976742\n",
      "[3800]\tvalid's auc: 0.956843\ttrain's auc: 0.978373\n",
      "[4000]\tvalid's auc: 0.9582\ttrain's auc: 0.979838\n",
      "[4200]\tvalid's auc: 0.959458\ttrain's auc: 0.981208\n",
      "[4400]\tvalid's auc: 0.960627\ttrain's auc: 0.982459\n",
      "[4600]\tvalid's auc: 0.961773\ttrain's auc: 0.983632\n",
      "[4800]\tvalid's auc: 0.962837\ttrain's auc: 0.984715\n",
      "[5000]\tvalid's auc: 0.963815\ttrain's auc: 0.985757\n",
      "[5200]\tvalid's auc: 0.964753\ttrain's auc: 0.98668\n",
      "[5400]\tvalid's auc: 0.965696\ttrain's auc: 0.987586\n",
      "[5600]\tvalid's auc: 0.966547\ttrain's auc: 0.988393\n",
      "[5800]\tvalid's auc: 0.967437\ttrain's auc: 0.989219\n",
      "[6000]\tvalid's auc: 0.968252\ttrain's auc: 0.989956\n",
      "[6200]\tvalid's auc: 0.968973\ttrain's auc: 0.990594\n",
      "[6400]\tvalid's auc: 0.969765\ttrain's auc: 0.991261\n",
      "[6600]\tvalid's auc: 0.970419\ttrain's auc: 0.991797\n",
      "[6800]\tvalid's auc: 0.971154\ttrain's auc: 0.992374\n",
      "[7000]\tvalid's auc: 0.971751\ttrain's auc: 0.992829\n",
      "[7200]\tvalid's auc: 0.97231\ttrain's auc: 0.993247\n",
      "[7400]\tvalid's auc: 0.972847\ttrain's auc: 0.993655\n",
      "[7600]\tvalid's auc: 0.973387\ttrain's auc: 0.994044\n",
      "[7800]\tvalid's auc: 0.973873\ttrain's auc: 0.994398\n",
      "[8000]\tvalid's auc: 0.974341\ttrain's auc: 0.994737\n",
      "[8200]\tvalid's auc: 0.974861\ttrain's auc: 0.995076\n",
      "[8400]\tvalid's auc: 0.975312\ttrain's auc: 0.99538\n",
      "[8600]\tvalid's auc: 0.975771\ttrain's auc: 0.995657\n",
      "[8800]\tvalid's auc: 0.976223\ttrain's auc: 0.995943\n",
      "[9000]\tvalid's auc: 0.976643\ttrain's auc: 0.996221\n",
      "[9200]\tvalid's auc: 0.977025\ttrain's auc: 0.996464\n",
      "[9400]\tvalid's auc: 0.977432\ttrain's auc: 0.996703\n",
      "[9600]\tvalid's auc: 0.977786\ttrain's auc: 0.996915\n",
      "[9800]\tvalid's auc: 0.978171\ttrain's auc: 0.997126\n",
      "[10000]\tvalid's auc: 0.978484\ttrain's auc: 0.997307\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.978484\ttrain's auc: 0.997307\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.883017\ttrain's auc: 0.885717\n",
      "[400]\tvalid's auc: 0.904193\ttrain's auc: 0.909089\n",
      "[600]\tvalid's auc: 0.91334\ttrain's auc: 0.920541\n",
      "[800]\tvalid's auc: 0.91939\ttrain's auc: 0.928736\n",
      "[1000]\tvalid's auc: 0.924302\ttrain's auc: 0.935238\n",
      "[1200]\tvalid's auc: 0.928872\ttrain's auc: 0.941193\n",
      "[1400]\tvalid's auc: 0.932575\ttrain's auc: 0.946118\n",
      "[1600]\tvalid's auc: 0.935988\ttrain's auc: 0.950473\n",
      "[1800]\tvalid's auc: 0.938923\ttrain's auc: 0.95441\n",
      "[2000]\tvalid's auc: 0.941591\ttrain's auc: 0.957984\n",
      "[2200]\tvalid's auc: 0.943965\ttrain's auc: 0.961054\n",
      "[2400]\tvalid's auc: 0.94628\ttrain's auc: 0.964082\n",
      "[2600]\tvalid's auc: 0.94831\ttrain's auc: 0.966736\n",
      "[2800]\tvalid's auc: 0.95006\ttrain's auc: 0.968947\n",
      "[3000]\tvalid's auc: 0.951961\ttrain's auc: 0.971258\n",
      "[3200]\tvalid's auc: 0.953427\ttrain's auc: 0.973211\n",
      "[3400]\tvalid's auc: 0.954905\ttrain's auc: 0.975018\n",
      "[3600]\tvalid's auc: 0.956297\ttrain's auc: 0.976675\n",
      "[3800]\tvalid's auc: 0.957619\ttrain's auc: 0.978268\n",
      "[4000]\tvalid's auc: 0.958963\ttrain's auc: 0.979803\n",
      "[4200]\tvalid's auc: 0.960185\ttrain's auc: 0.981179\n",
      "[4400]\tvalid's auc: 0.961319\ttrain's auc: 0.982415\n",
      "[4600]\tvalid's auc: 0.96244\ttrain's auc: 0.983632\n",
      "[4800]\tvalid's auc: 0.963425\ttrain's auc: 0.984716\n",
      "[5000]\tvalid's auc: 0.964353\ttrain's auc: 0.985634\n",
      "[5200]\tvalid's auc: 0.965258\ttrain's auc: 0.986578\n",
      "[5400]\tvalid's auc: 0.966165\ttrain's auc: 0.987421\n",
      "[5600]\tvalid's auc: 0.967032\ttrain's auc: 0.988256\n",
      "[5800]\tvalid's auc: 0.967842\ttrain's auc: 0.988993\n",
      "[6000]\tvalid's auc: 0.968646\ttrain's auc: 0.989702\n",
      "[6200]\tvalid's auc: 0.969402\ttrain's auc: 0.990396\n",
      "[6400]\tvalid's auc: 0.970092\ttrain's auc: 0.991001\n",
      "[6600]\tvalid's auc: 0.970753\ttrain's auc: 0.991574\n",
      "[6800]\tvalid's auc: 0.971387\ttrain's auc: 0.99209\n",
      "[7000]\tvalid's auc: 0.971952\ttrain's auc: 0.992585\n",
      "[7200]\tvalid's auc: 0.972516\ttrain's auc: 0.993016\n",
      "[7400]\tvalid's auc: 0.973113\ttrain's auc: 0.993477\n",
      "[7600]\tvalid's auc: 0.973718\ttrain's auc: 0.993914\n",
      "[7800]\tvalid's auc: 0.97422\ttrain's auc: 0.994311\n",
      "[8000]\tvalid's auc: 0.97473\ttrain's auc: 0.994681\n",
      "[8200]\tvalid's auc: 0.975235\ttrain's auc: 0.99503\n",
      "[8400]\tvalid's auc: 0.975729\ttrain's auc: 0.995358\n",
      "[8600]\tvalid's auc: 0.976181\ttrain's auc: 0.995641\n",
      "[8800]\tvalid's auc: 0.97658\ttrain's auc: 0.995924\n",
      "[9000]\tvalid's auc: 0.976996\ttrain's auc: 0.99619\n",
      "[9200]\tvalid's auc: 0.977384\ttrain's auc: 0.996424\n",
      "[9400]\tvalid's auc: 0.977797\ttrain's auc: 0.996681\n",
      "[9600]\tvalid's auc: 0.978182\ttrain's auc: 0.996906\n",
      "[9800]\tvalid's auc: 0.978532\ttrain's auc: 0.99709\n",
      "[10000]\tvalid's auc: 0.978879\ttrain's auc: 0.997296\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.978879\ttrain's auc: 0.997296\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.883707\ttrain's auc: 0.885774\n",
      "[400]\tvalid's auc: 0.905682\ttrain's auc: 0.909166\n",
      "[600]\tvalid's auc: 0.915213\ttrain's auc: 0.921009\n",
      "[800]\tvalid's auc: 0.921136\ttrain's auc: 0.929149\n",
      "[1000]\tvalid's auc: 0.926153\ttrain's auc: 0.935904\n",
      "[1200]\tvalid's auc: 0.930206\ttrain's auc: 0.941347\n",
      "[1400]\tvalid's auc: 0.933714\ttrain's auc: 0.946092\n",
      "[1600]\tvalid's auc: 0.936893\ttrain's auc: 0.950454\n",
      "[1800]\tvalid's auc: 0.939668\ttrain's auc: 0.954216\n",
      "[2000]\tvalid's auc: 0.942432\ttrain's auc: 0.957767\n",
      "[2200]\tvalid's auc: 0.944629\ttrain's auc: 0.960795\n",
      "[2400]\tvalid's auc: 0.946983\ttrain's auc: 0.963927\n",
      "[2600]\tvalid's auc: 0.949017\ttrain's auc: 0.966526\n",
      "[2800]\tvalid's auc: 0.950923\ttrain's auc: 0.968946\n",
      "[3000]\tvalid's auc: 0.952773\ttrain's auc: 0.971237\n",
      "[3200]\tvalid's auc: 0.954341\ttrain's auc: 0.973168\n",
      "[3400]\tvalid's auc: 0.955888\ttrain's auc: 0.975079\n",
      "[3600]\tvalid's auc: 0.957296\ttrain's auc: 0.976699\n",
      "[3800]\tvalid's auc: 0.958582\ttrain's auc: 0.978281\n",
      "[4000]\tvalid's auc: 0.959912\ttrain's auc: 0.979748\n",
      "[4200]\tvalid's auc: 0.961095\ttrain's auc: 0.981084\n",
      "[4400]\tvalid's auc: 0.962189\ttrain's auc: 0.982315\n",
      "[4600]\tvalid's auc: 0.963303\ttrain's auc: 0.983534\n",
      "[4800]\tvalid's auc: 0.964342\ttrain's auc: 0.98464\n",
      "[5000]\tvalid's auc: 0.965364\ttrain's auc: 0.985756\n",
      "[5200]\tvalid's auc: 0.966298\ttrain's auc: 0.986693\n",
      "[5400]\tvalid's auc: 0.967212\ttrain's auc: 0.987603\n",
      "[5600]\tvalid's auc: 0.968027\ttrain's auc: 0.988387\n",
      "[5800]\tvalid's auc: 0.968828\ttrain's auc: 0.989159\n",
      "[6000]\tvalid's auc: 0.969615\ttrain's auc: 0.989883\n",
      "[6200]\tvalid's auc: 0.970358\ttrain's auc: 0.990532\n",
      "[6400]\tvalid's auc: 0.971026\ttrain's auc: 0.991124\n",
      "[6600]\tvalid's auc: 0.971633\ttrain's auc: 0.991653\n",
      "[6800]\tvalid's auc: 0.972239\ttrain's auc: 0.992181\n",
      "[7000]\tvalid's auc: 0.972856\ttrain's auc: 0.992686\n",
      "[7200]\tvalid's auc: 0.973433\ttrain's auc: 0.993135\n",
      "[7400]\tvalid's auc: 0.974012\ttrain's auc: 0.993577\n",
      "[7600]\tvalid's auc: 0.974562\ttrain's auc: 0.994005\n",
      "[7800]\tvalid's auc: 0.975059\ttrain's auc: 0.994371\n",
      "[8000]\tvalid's auc: 0.975523\ttrain's auc: 0.994713\n",
      "[8200]\tvalid's auc: 0.976009\ttrain's auc: 0.995038\n",
      "[8400]\tvalid's auc: 0.976466\ttrain's auc: 0.995337\n",
      "[8600]\tvalid's auc: 0.976937\ttrain's auc: 0.99562\n",
      "[8800]\tvalid's auc: 0.977343\ttrain's auc: 0.995907\n",
      "[9000]\tvalid's auc: 0.977756\ttrain's auc: 0.996171\n",
      "[9200]\tvalid's auc: 0.978163\ttrain's auc: 0.996428\n",
      "[9400]\tvalid's auc: 0.978528\ttrain's auc: 0.996643\n",
      "[9600]\tvalid's auc: 0.978889\ttrain's auc: 0.996871\n",
      "[9800]\tvalid's auc: 0.979247\ttrain's auc: 0.997095\n",
      "[10000]\tvalid's auc: 0.979602\ttrain's auc: 0.99729\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.979602\ttrain's auc: 0.99729\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.883405\ttrain's auc: 0.886924\n",
      "[400]\tvalid's auc: 0.904602\ttrain's auc: 0.909718\n",
      "[600]\tvalid's auc: 0.914124\ttrain's auc: 0.921228\n",
      "[800]\tvalid's auc: 0.920456\ttrain's auc: 0.929515\n",
      "[1000]\tvalid's auc: 0.925572\ttrain's auc: 0.936127\n",
      "[1200]\tvalid's auc: 0.929968\ttrain's auc: 0.941919\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1400]\tvalid's auc: 0.933834\ttrain's auc: 0.947027\n",
      "[1600]\tvalid's auc: 0.936959\ttrain's auc: 0.951171\n",
      "[1800]\tvalid's auc: 0.939937\ttrain's auc: 0.955142\n",
      "[2000]\tvalid's auc: 0.942512\ttrain's auc: 0.958648\n",
      "[2200]\tvalid's auc: 0.944704\ttrain's auc: 0.96166\n",
      "[2400]\tvalid's auc: 0.946852\ttrain's auc: 0.964516\n",
      "[2600]\tvalid's auc: 0.948759\ttrain's auc: 0.967\n",
      "[2800]\tvalid's auc: 0.95062\ttrain's auc: 0.969423\n",
      "[3000]\tvalid's auc: 0.95232\ttrain's auc: 0.971447\n",
      "[3200]\tvalid's auc: 0.953929\ttrain's auc: 0.973521\n",
      "[3400]\tvalid's auc: 0.955465\ttrain's auc: 0.975352\n",
      "[3600]\tvalid's auc: 0.956973\ttrain's auc: 0.977105\n",
      "[3800]\tvalid's auc: 0.958301\ttrain's auc: 0.978653\n",
      "[4000]\tvalid's auc: 0.959412\ttrain's auc: 0.979989\n",
      "[4200]\tvalid's auc: 0.960601\ttrain's auc: 0.981302\n",
      "[4400]\tvalid's auc: 0.961791\ttrain's auc: 0.982629\n",
      "[4600]\tvalid's auc: 0.962844\ttrain's auc: 0.983768\n",
      "[4800]\tvalid's auc: 0.963909\ttrain's auc: 0.984856\n",
      "[5000]\tvalid's auc: 0.964882\ttrain's auc: 0.985816\n",
      "[5200]\tvalid's auc: 0.965716\ttrain's auc: 0.986674\n",
      "[5400]\tvalid's auc: 0.966584\ttrain's auc: 0.987558\n",
      "[5600]\tvalid's auc: 0.967353\ttrain's auc: 0.988286\n",
      "[5800]\tvalid's auc: 0.96812\ttrain's auc: 0.989061\n",
      "[6000]\tvalid's auc: 0.968878\ttrain's auc: 0.989764\n",
      "[6200]\tvalid's auc: 0.969647\ttrain's auc: 0.990466\n",
      "[6400]\tvalid's auc: 0.970332\ttrain's auc: 0.991076\n",
      "[6600]\tvalid's auc: 0.970966\ttrain's auc: 0.991667\n",
      "[6800]\tvalid's auc: 0.971615\ttrain's auc: 0.992219\n",
      "[7000]\tvalid's auc: 0.972251\ttrain's auc: 0.992763\n",
      "[7200]\tvalid's auc: 0.972814\ttrain's auc: 0.993212\n",
      "[7400]\tvalid's auc: 0.973412\ttrain's auc: 0.993651\n",
      "[7600]\tvalid's auc: 0.973963\ttrain's auc: 0.994057\n",
      "[7800]\tvalid's auc: 0.97446\ttrain's auc: 0.994397\n",
      "[8000]\tvalid's auc: 0.974963\ttrain's auc: 0.99476\n",
      "[8200]\tvalid's auc: 0.975415\ttrain's auc: 0.99508\n",
      "[8400]\tvalid's auc: 0.975858\ttrain's auc: 0.995382\n",
      "[8600]\tvalid's auc: 0.976281\ttrain's auc: 0.995654\n",
      "[8800]\tvalid's auc: 0.976693\ttrain's auc: 0.995933\n",
      "[9000]\tvalid's auc: 0.977073\ttrain's auc: 0.996196\n",
      "[9200]\tvalid's auc: 0.977459\ttrain's auc: 0.996428\n",
      "[9400]\tvalid's auc: 0.977842\ttrain's auc: 0.996657\n",
      "[9600]\tvalid's auc: 0.97821\ttrain's auc: 0.996871\n",
      "[9800]\tvalid's auc: 0.978544\ttrain's auc: 0.997058\n",
      "[10000]\tvalid's auc: 0.978893\ttrain's auc: 0.997255\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.978893\ttrain's auc: 0.997255\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.882555\ttrain's auc: 0.885647\n",
      "[400]\tvalid's auc: 0.90501\ttrain's auc: 0.90968\n",
      "[600]\tvalid's auc: 0.914056\ttrain's auc: 0.921072\n",
      "[800]\tvalid's auc: 0.920324\ttrain's auc: 0.929263\n",
      "[1000]\tvalid's auc: 0.925086\ttrain's auc: 0.935555\n",
      "[1200]\tvalid's auc: 0.92914\ttrain's auc: 0.94105\n",
      "[1400]\tvalid's auc: 0.932924\ttrain's auc: 0.945986\n",
      "[1600]\tvalid's auc: 0.936275\ttrain's auc: 0.950351\n",
      "[1800]\tvalid's auc: 0.93925\ttrain's auc: 0.954274\n",
      "[2000]\tvalid's auc: 0.941854\ttrain's auc: 0.957687\n",
      "[2200]\tvalid's auc: 0.944227\ttrain's auc: 0.960911\n",
      "[2400]\tvalid's auc: 0.946512\ttrain's auc: 0.96384\n",
      "[2600]\tvalid's auc: 0.948493\ttrain's auc: 0.966344\n",
      "[2800]\tvalid's auc: 0.950352\ttrain's auc: 0.968602\n",
      "[3000]\tvalid's auc: 0.952226\ttrain's auc: 0.970923\n",
      "[3200]\tvalid's auc: 0.953867\ttrain's auc: 0.972911\n",
      "[3400]\tvalid's auc: 0.955332\ttrain's auc: 0.974746\n",
      "[3600]\tvalid's auc: 0.95674\ttrain's auc: 0.976423\n",
      "[3800]\tvalid's auc: 0.958029\ttrain's auc: 0.977964\n",
      "[4000]\tvalid's auc: 0.959248\ttrain's auc: 0.979357\n",
      "[4200]\tvalid's auc: 0.960502\ttrain's auc: 0.980743\n",
      "[4400]\tvalid's auc: 0.961685\ttrain's auc: 0.982034\n",
      "[4600]\tvalid's auc: 0.962841\ttrain's auc: 0.983252\n",
      "[4800]\tvalid's auc: 0.963868\ttrain's auc: 0.984311\n",
      "[5000]\tvalid's auc: 0.96488\ttrain's auc: 0.98533\n",
      "[5200]\tvalid's auc: 0.965848\ttrain's auc: 0.98635\n",
      "[5400]\tvalid's auc: 0.966773\ttrain's auc: 0.987291\n",
      "[5600]\tvalid's auc: 0.967618\ttrain's auc: 0.988115\n",
      "[5800]\tvalid's auc: 0.96842\ttrain's auc: 0.988863\n",
      "[6000]\tvalid's auc: 0.969156\ttrain's auc: 0.989557\n",
      "[6200]\tvalid's auc: 0.969898\ttrain's auc: 0.990219\n",
      "[6400]\tvalid's auc: 0.970598\ttrain's auc: 0.990841\n",
      "[6600]\tvalid's auc: 0.971305\ttrain's auc: 0.991492\n",
      "[6800]\tvalid's auc: 0.971934\ttrain's auc: 0.992031\n",
      "[7000]\tvalid's auc: 0.972533\ttrain's auc: 0.99257\n",
      "[7200]\tvalid's auc: 0.973118\ttrain's auc: 0.993034\n",
      "[7400]\tvalid's auc: 0.973649\ttrain's auc: 0.99349\n",
      "[7600]\tvalid's auc: 0.974177\ttrain's auc: 0.993903\n",
      "[7800]\tvalid's auc: 0.974707\ttrain's auc: 0.994288\n",
      "[8000]\tvalid's auc: 0.975186\ttrain's auc: 0.994627\n",
      "[8200]\tvalid's auc: 0.975666\ttrain's auc: 0.994991\n",
      "[8400]\tvalid's auc: 0.976193\ttrain's auc: 0.995337\n",
      "[8600]\tvalid's auc: 0.976657\ttrain's auc: 0.995655\n",
      "[8800]\tvalid's auc: 0.977094\ttrain's auc: 0.99597\n",
      "[9000]\tvalid's auc: 0.977547\ttrain's auc: 0.996261\n",
      "[9200]\tvalid's auc: 0.977946\ttrain's auc: 0.996501\n",
      "[9400]\tvalid's auc: 0.978315\ttrain's auc: 0.996731\n",
      "[9600]\tvalid's auc: 0.978672\ttrain's auc: 0.996939\n",
      "[9800]\tvalid's auc: 0.979025\ttrain's auc: 0.997145\n",
      "[10000]\tvalid's auc: 0.979334\ttrain's auc: 0.997336\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.979334\ttrain's auc: 0.997336\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.882529\ttrain's auc: 0.886331\n",
      "[400]\tvalid's auc: 0.903578\ttrain's auc: 0.909156\n",
      "[600]\tvalid's auc: 0.913031\ttrain's auc: 0.920843\n",
      "[800]\tvalid's auc: 0.919087\ttrain's auc: 0.928783\n",
      "[1000]\tvalid's auc: 0.924216\ttrain's auc: 0.935607\n",
      "[1200]\tvalid's auc: 0.928426\ttrain's auc: 0.941321\n",
      "[1400]\tvalid's auc: 0.932085\ttrain's auc: 0.946275\n",
      "[1600]\tvalid's auc: 0.935353\ttrain's auc: 0.950646\n",
      "[1800]\tvalid's auc: 0.938442\ttrain's auc: 0.954747\n",
      "[2000]\tvalid's auc: 0.941281\ttrain's auc: 0.958306\n",
      "[2200]\tvalid's auc: 0.94374\ttrain's auc: 0.961329\n",
      "[2400]\tvalid's auc: 0.946187\ttrain's auc: 0.9643\n",
      "[2600]\tvalid's auc: 0.948125\ttrain's auc: 0.966779\n",
      "[2800]\tvalid's auc: 0.950003\ttrain's auc: 0.96909\n",
      "[3000]\tvalid's auc: 0.951839\ttrain's auc: 0.971321\n",
      "[3200]\tvalid's auc: 0.953654\ttrain's auc: 0.973516\n",
      "[3400]\tvalid's auc: 0.955201\ttrain's auc: 0.975285\n",
      "[3600]\tvalid's auc: 0.956571\ttrain's auc: 0.976879\n",
      "[3800]\tvalid's auc: 0.958082\ttrain's auc: 0.978514\n",
      "[4000]\tvalid's auc: 0.959339\ttrain's auc: 0.979923\n",
      "[4200]\tvalid's auc: 0.960573\ttrain's auc: 0.981306\n",
      "[4400]\tvalid's auc: 0.961843\ttrain's auc: 0.982591\n",
      "[4600]\tvalid's auc: 0.962911\ttrain's auc: 0.983746\n",
      "[4800]\tvalid's auc: 0.963947\ttrain's auc: 0.984789\n",
      "[5000]\tvalid's auc: 0.964905\ttrain's auc: 0.98582\n",
      "[5200]\tvalid's auc: 0.965834\ttrain's auc: 0.986758\n",
      "[5400]\tvalid's auc: 0.966781\ttrain's auc: 0.98766\n",
      "[5600]\tvalid's auc: 0.967611\ttrain's auc: 0.988454\n",
      "[5800]\tvalid's auc: 0.968389\ttrain's auc: 0.989212\n",
      "[6000]\tvalid's auc: 0.969141\ttrain's auc: 0.989874\n",
      "[6200]\tvalid's auc: 0.969869\ttrain's auc: 0.990499\n",
      "[6400]\tvalid's auc: 0.970585\ttrain's auc: 0.991116\n",
      "[6600]\tvalid's auc: 0.971268\ttrain's auc: 0.991696\n",
      "[6800]\tvalid's auc: 0.971905\ttrain's auc: 0.992214\n",
      "[7000]\tvalid's auc: 0.972494\ttrain's auc: 0.992723\n",
      "[7200]\tvalid's auc: 0.973104\ttrain's auc: 0.993225\n",
      "[7400]\tvalid's auc: 0.973635\ttrain's auc: 0.993633\n",
      "[7600]\tvalid's auc: 0.974145\ttrain's auc: 0.994037\n",
      "[7800]\tvalid's auc: 0.974655\ttrain's auc: 0.994414\n",
      "[8000]\tvalid's auc: 0.975193\ttrain's auc: 0.994777\n",
      "[8200]\tvalid's auc: 0.975694\ttrain's auc: 0.995109\n",
      "[8400]\tvalid's auc: 0.976176\ttrain's auc: 0.995429\n",
      "[8600]\tvalid's auc: 0.976668\ttrain's auc: 0.995745\n",
      "[8800]\tvalid's auc: 0.97709\ttrain's auc: 0.996017\n",
      "[9000]\tvalid's auc: 0.977574\ttrain's auc: 0.996328\n",
      "[9200]\tvalid's auc: 0.977963\ttrain's auc: 0.996555\n",
      "[9400]\tvalid's auc: 0.978364\ttrain's auc: 0.996775\n",
      "[9600]\tvalid's auc: 0.978734\ttrain's auc: 0.996991\n",
      "[9800]\tvalid's auc: 0.979119\ttrain's auc: 0.99719\n",
      "[10000]\tvalid's auc: 0.979461\ttrain's auc: 0.997357\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[10000]\tvalid's auc: 0.979461\ttrain's auc: 0.997357\n",
      "Training Data Shape:  (49650, 50)\n",
      "Testing Data Shape:  (48744, 50)\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.61561\ttrain's auc: 0.733137\n",
      "Early stopping, best iteration is:\n",
      "[190]\tvalid's auc: 0.616802\ttrain's auc: 0.729489\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.637649\ttrain's auc: 0.7313\n",
      "Early stopping, best iteration is:\n",
      "[223]\tvalid's auc: 0.638234\ttrain's auc: 0.739835\n",
      "Training until validation scores don't improve for 100 rounds.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200]\tvalid's auc: 0.624199\ttrain's auc: 0.732056\n",
      "Early stopping, best iteration is:\n",
      "[137]\tvalid's auc: 0.625451\ttrain's auc: 0.705626\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[68]\tvalid's auc: 0.617607\ttrain's auc: 0.672786\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.606953\ttrain's auc: 0.732045\n",
      "Early stopping, best iteration is:\n",
      "[128]\tvalid's auc: 0.608358\ttrain's auc: 0.702488\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.63116\ttrain's auc: 0.731104\n",
      "Early stopping, best iteration is:\n",
      "[266]\tvalid's auc: 0.631938\ttrain's auc: 0.754708\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.621315\ttrain's auc: 0.73385\n",
      "Early stopping, best iteration is:\n",
      "[272]\tvalid's auc: 0.622932\ttrain's auc: 0.758955\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.614512\ttrain's auc: 0.731293\n",
      "Early stopping, best iteration is:\n",
      "[102]\tvalid's auc: 0.615479\ttrain's auc: 0.690047\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[86]\tvalid's auc: 0.619822\ttrain's auc: 0.681282\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.609125\ttrain's auc: 0.732619\n",
      "Early stopping, best iteration is:\n",
      "[112]\tvalid's auc: 0.610624\ttrain's auc: 0.695125\n",
      "Training Data Shape:  (565372, 250)\n",
      "Testing Data Shape:  (48744, 250)\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.97947\ttrain's auc: 0.980083\n",
      "[400]\tvalid's auc: 0.979934\ttrain's auc: 0.984395\n",
      "[600]\tvalid's auc: 0.980007\ttrain's auc: 0.987527\n",
      "Early stopping, best iteration is:\n",
      "[648]\tvalid's auc: 0.980038\ttrain's auc: 0.988154\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977038\ttrain's auc: 0.98033\n",
      "[400]\tvalid's auc: 0.97738\ttrain's auc: 0.984515\n",
      "Early stopping, best iteration is:\n",
      "[332]\tvalid's auc: 0.977411\ttrain's auc: 0.983223\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977976\ttrain's auc: 0.980243\n",
      "[400]\tvalid's auc: 0.978413\ttrain's auc: 0.984386\n",
      "[600]\tvalid's auc: 0.97846\ttrain's auc: 0.987517\n",
      "Early stopping, best iteration is:\n",
      "[548]\tvalid's auc: 0.9785\ttrain's auc: 0.98679\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.97863\ttrain's auc: 0.980181\n",
      "[400]\tvalid's auc: 0.979025\ttrain's auc: 0.984351\n",
      "[600]\tvalid's auc: 0.9791\ttrain's auc: 0.987494\n",
      "Early stopping, best iteration is:\n",
      "[552]\tvalid's auc: 0.979128\ttrain's auc: 0.98681\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977838\ttrain's auc: 0.980214\n",
      "[400]\tvalid's auc: 0.978254\ttrain's auc: 0.984466\n",
      "Early stopping, best iteration is:\n",
      "[423]\tvalid's auc: 0.978285\ttrain's auc: 0.984892\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.97797\ttrain's auc: 0.980204\n",
      "[400]\tvalid's auc: 0.978491\ttrain's auc: 0.984511\n",
      "[600]\tvalid's auc: 0.978525\ttrain's auc: 0.987632\n",
      "Early stopping, best iteration is:\n",
      "[570]\tvalid's auc: 0.978538\ttrain's auc: 0.987235\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.978269\ttrain's auc: 0.980204\n",
      "[400]\tvalid's auc: 0.978793\ttrain's auc: 0.984393\n",
      "[600]\tvalid's auc: 0.978869\ttrain's auc: 0.98751\n",
      "[800]\tvalid's auc: 0.978838\ttrain's auc: 0.989901\n",
      "Early stopping, best iteration is:\n",
      "[711]\tvalid's auc: 0.978895\ttrain's auc: 0.988949\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977624\ttrain's auc: 0.980237\n",
      "[400]\tvalid's auc: 0.977986\ttrain's auc: 0.984483\n",
      "Early stopping, best iteration is:\n",
      "[401]\tvalid's auc: 0.977988\ttrain's auc: 0.9845\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.978279\ttrain's auc: 0.980176\n",
      "[400]\tvalid's auc: 0.978618\ttrain's auc: 0.984372\n",
      "[600]\tvalid's auc: 0.978669\ttrain's auc: 0.987481\n",
      "Early stopping, best iteration is:\n",
      "[559]\tvalid's auc: 0.978685\ttrain's auc: 0.986949\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.977914\ttrain's auc: 0.980242\n",
      "[400]\tvalid's auc: 0.978395\ttrain's auc: 0.984426\n",
      "[600]\tvalid's auc: 0.978473\ttrain's auc: 0.987581\n",
      "Early stopping, best iteration is:\n",
      "[578]\tvalid's auc: 0.978494\ttrain's auc: 0.987276\n",
      "Training Data Shape:  (49650, 250)\n",
      "Testing Data Shape:  (48744, 250)\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.759971\ttrain's auc: 0.842635\n",
      "[400]\tvalid's auc: 0.761946\ttrain's auc: 0.890545\n",
      "Early stopping, best iteration is:\n",
      "[454]\tvalid's auc: 0.762146\ttrain's auc: 0.900654\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.778433\ttrain's auc: 0.841139\n",
      "[400]\tvalid's auc: 0.779597\ttrain's auc: 0.889288\n",
      "Early stopping, best iteration is:\n",
      "[429]\tvalid's auc: 0.780222\ttrain's auc: 0.89484\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.775408\ttrain's auc: 0.841207\n",
      "[400]\tvalid's auc: 0.776906\ttrain's auc: 0.88895\n",
      "Early stopping, best iteration is:\n",
      "[420]\tvalid's auc: 0.776995\ttrain's auc: 0.892898\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.770064\ttrain's auc: 0.841414\n",
      "Early stopping, best iteration is:\n",
      "[259]\tvalid's auc: 0.77082\ttrain's auc: 0.857344\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.776777\ttrain's auc: 0.841512\n",
      "Early stopping, best iteration is:\n",
      "[182]\tvalid's auc: 0.776985\ttrain's auc: 0.83612\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.7821\ttrain's auc: 0.840754\n",
      "[400]\tvalid's auc: 0.782727\ttrain's auc: 0.889022\n",
      "Early stopping, best iteration is:\n",
      "[339]\tvalid's auc: 0.783299\ttrain's auc: 0.876343\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.772521\ttrain's auc: 0.841049\n",
      "Early stopping, best iteration is:\n",
      "[296]\tvalid's auc: 0.773119\ttrain's auc: 0.866742\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.766528\ttrain's auc: 0.841231\n",
      "[400]\tvalid's auc: 0.767448\ttrain's auc: 0.888355\n",
      "[600]\tvalid's auc: 0.768785\ttrain's auc: 0.92269\n",
      "[800]\tvalid's auc: 0.768971\ttrain's auc: 0.946698\n",
      "Early stopping, best iteration is:\n",
      "[797]\tvalid's auc: 0.769087\ttrain's auc: 0.946397\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.772683\ttrain's auc: 0.840861\n",
      "Early stopping, best iteration is:\n",
      "[203]\tvalid's auc: 0.772837\ttrain's auc: 0.841673\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[200]\tvalid's auc: 0.761803\ttrain's auc: 0.842328\n",
      "Early stopping, best iteration is:\n",
      "[258]\tvalid's auc: 0.762496\ttrain's auc: 0.857905\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(trainsets)):\n",
    "    submission, fi, metrics = model(trainsets[i], testsets[i], n_estimators=10000, learning_rate = 0.05, \n",
    "                                reg_alpha = 0.1, reg_lambda = 0.1)\n",
    "    submission.to_csv('results2/submission_' + result_names[i] + '.csv', index = False)\n",
    "    fi.to_csv('results2/fi_' + result_names[i] + '.csv', index = False)\n",
    "    metrics.to_csv('results2/metrics_' + result_names[i] + '.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tekkredi]",
   "language": "python",
   "name": "conda-env-tekkredi-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}