Alice.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "import os\n",
    "import gc\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.cross_validation import train_test_split  \n",
    "import sys\n",
    "import seaborn as sns\n",
    "import random\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import KFold\n",
    "from itertools import combinations\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import operator  \n",
    "from sklearn.metrics import roc_curve,auc\n",
    "import datetime\n",
    "from lib import interaction_features,proj_num_on_cat,cat_count, convertToOne, set_missing\n",
    "from skopt import BayesSearchCV\n",
    "from sklearn import metrics\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from Xgboost_Feature import XgboostFeature\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "from imputer import Imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Xgboost Feature start, new_feature number: 30\n"
     ]
    }
   ],
   "source": [
    "xb = XgboostFeature()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.append('/Users/lin/Enterprise/missingno-master/missingno')\n",
    "import missingno as msno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "startID = ['f20','f24','f28','f32','f36','f48','f52','f54',\n",
    "           'f64','f72','f76','f102','f107','f111','f155','f161','f166',\n",
    "          'f211','f254','f278']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "catid = ['f1','f2','f3','f4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "userid = ['f6','f7','f8','f9','f10','f11','f12','f13','f14','f15','f16','f17','f18','f19']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "part = []\n",
    "for i in range(166,278):\n",
    "        part.append(\"f\" + str(i))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split remain columns into 3 groups!\n"
     ]
    }
   ],
   "source": [
    "print(\"Split remain columns into 3 groups!\")\n",
    "cat_list = ['f1','f2','f3','f4','f6','f7','f8','f9','f10','f11','f12','f13','f14','f15','f16','f17','f18','f19']#分类性\n",
    "reg_list = ['f182','f183','f184','f185','f204','f205','f206','f207','f259','f260','f261','f270','f271']#离散性\n",
    "ind_list = ['f161','f162','f163','f164','f165']#连续性\n",
    "for i in range(211,254):\n",
    "        ind_list.append(\"f\" + str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createCol():\n",
    "    col = ['label','date']\n",
    "    for i in range(1,298):\n",
    "        col.append(\"f\" + str(i))\n",
    "    \n",
    "    return col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def creatDtype():\n",
    "    dtype = {\n",
    "        'id':'object',\n",
    "        'label':'int8',\n",
    "        'date':'int64',\n",
    "        'f1' : 'uint8',\n",
    "        'f2' : 'uint8',\n",
    "        'f3' : 'uint8',\n",
    "        'f4' : 'uint8',\n",
    "        'f5' : 'float32'\n",
    "    }\n",
    "    for i in range(20,298):\n",
    "        dtype[\"f\" + str(i)] = 'float32'\n",
    "    for i in range(6,20):\n",
    "        dtype[\"f\" + str(i)] = 'uint8'\n",
    "    \n",
    "    return dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_list = []\n",
    "for i in range(161,278):\n",
    "    col_list.append(\"f\"+str(i))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading data!\n",
      "(994731, 299)\n"
     ]
    }
   ],
   "source": [
    "print(\"loading data!\")\n",
    "train = pd.read_csv('atec_anti_fraud_train.csv', header = 0,dtype=creatDtype(),usecols=createCol())\n",
    "test = pd.read_csv('atec_anti_fraud_test_b.csv',header = 0,dtype=creatDtype())\n",
    "#test1 = pd.read_csv('atec_anti_fraud_test_b.csv',header = 0,dtype=creatDtype())\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = train.sort_values(by=['date'])\n",
    "# train = train[train.date < 20171016]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train = train.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Delete -1 in label!\")\n",
    "# train.drop(np.where(train.label == -1)[0],inplace=True)\n",
    "# train = train.reset_index(drop = True)\n",
    "# print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.replace(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test['label'] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Take off label!\n",
      "(994731, 298)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Take off label!\")\n",
    "label = train['label']\n",
    "train = train.drop(['label'],axis=1)\n",
    "print(train.shape)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Concat train and test!\n",
      "(1495269, 298)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Concat train and test!\")\n",
    "len_train = len(train)\n",
    "test = test.drop(['id'],axis=1)\n",
    "train = train.append(test)\n",
    "train = train.reset_index(drop=True)\n",
    "print(train.shape)\n",
    "del test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counting f82!\n",
      "Merging!\n",
      "(1495269, 299)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "206"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Counting f82!\")\n",
    "gp = train[userid + ['f82']].groupby(by=userid)['f82'].count().reset_index().rename(columns={'f82': 'money_count'})\n",
    "print(\"Merging!\")\n",
    "train = train.merge(gp, on=userid, how='left')\n",
    "train['money_count'] = train['money_count'].astype('uint16')\n",
    "print(train.shape)\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count f1-f4!\n",
      "Merging!\n",
      "(1495269, 300)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "136"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Count f1-f4!\")\n",
    "train['f300'] = ''\n",
    "gp = train[['f1','f2','f3','f4','f300']].groupby(by=['f1','f2','f3','f4'])['f300'].count().reset_index().rename(columns={'f300': 'count1_4'})\n",
    "print(\"Merging!\")\n",
    "train = train.merge(gp, on=['f1','f2','f3','f4'], how='left')\n",
    "train['count1_4'] = train['count1_4'].astype('uint16')\n",
    "train = train.drop(['f300'],axis=1)\n",
    "print(train.shape)\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count f6-f19!\n",
      "Merging!\n",
      "(1495269, 301)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "240"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Count f6-f19!\")\n",
    "train['f300'] = ''\n",
    "gp = train[userid + ['f300']].groupby(userid)['f300'].count().reset_index().rename(columns={'f300': 'count6_19'})\n",
    "print(\"Merging!\")\n",
    "train = train.merge(gp, on=userid, how='left')\n",
    "train['count6_19'] = train['count6_19'].astype('uint16')\n",
    "train = train.drop(['f300'],axis=1)\n",
    "print(train.shape)\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Count f1-f19!\")\n",
    "# col = userid + ['f1','f2','f3','f4']\n",
    "# train['f300'] = ''\n",
    "# gp = train[col + ['f300']].groupby(col)['f300'].count().reset_index().rename(columns={'f300': 'count1_19'})\n",
    "# print(\"Merging!\")\n",
    "# train = train.merge(gp, on=col, how='left')\n",
    "# train['count1_19'] = train['count1_19'].astype('uint16')\n",
    "# train = train.drop(['f300'],axis=1)\n",
    "# print(train.shape)\n",
    "# del gp\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Counting 6-19 for each part!\")\n",
    "# for i in startID:\n",
    "#     gp = train[userid + [i]].groupby(by=userid)[i].count().reset_index().rename(columns={i: 'count6_19'+ i})\n",
    "#     train = train.merge(gp, on=userid, how='left')\n",
    "#     train['count6_19'+ i] = train['count6_19'+ i].astype('uint16')\n",
    "#     del gp\n",
    "#     gc.collect()\n",
    "# print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Counting 1-4 for each part!\")\n",
    "# for i in startID:\n",
    "#     gp = train[catid + [i]].groupby(by=catid)[i].count().reset_index().rename(columns={i: 'count1_4'+ i})\n",
    "#     train = train.merge(gp, on=catid, how='left')\n",
    "#     train['count1_4'+ i] = train['count1_4'+ i].astype('uint16')\n",
    "#     del gp\n",
    "#     gc.collect()\n",
    "# print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Counting 1-19 for each part!\")\n",
    "# allid = userid+catid\n",
    "# for i in startID:\n",
    "#     gp = train[allid + [i]].groupby(by=allid)[i].count().reset_index().rename(columns={i: 'count1_19'+ i})\n",
    "#     train = train.merge(gp, on=allid, how='left')\n",
    "#     train['count1_19'+ i] = train['count1_19'+ i].astype('uint16')\n",
    "#     del gp\n",
    "#     gc.collect()\n",
    "# print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculate numbers of Nan in each line\n",
      "(1495269, 302)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Calculate numbers of Nan in each line\")\n",
    "train['count_nan'] = train.T.isnull().sum()\n",
    "train['count_nan'] = train['count_nan'].astype('uint16')\n",
    "print(train.shape)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculate percent of Nan in each line!\n",
      "(1495269, 303)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Calculate percent of Nan in each line!\")\n",
    "train['nan_percent'] = train['count_nan'] / 300.0\n",
    "train['nan_percent'] = train['nan_percent'].astype('float32')\n",
    "print(train.shape)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delete col with too much NaN!\n",
      "('Delcol number is ', 127)\n",
      "(1495269, 176)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Delete col with too much NaN!\")\n",
    "delcol = []\n",
    "for i in train.columns:\n",
    "    if(float(train[i].isnull().sum()) / len(train) > 0.3):\n",
    "        delcol.append(i)\n",
    "print(\"Delcol number is \", len(delcol))\n",
    "train = train.drop(delcol,axis=1)\n",
    "del delcol\n",
    "print(train.shape)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Creating occurance features!\")\n",
    "# train = train.fillna(-1)\n",
    "# for i in col_list:\n",
    "#     print(\"deal with\",i)\n",
    "#     f_ratio = pd.DataFrame(train[i].value_counts())\n",
    "#     #f_ratio[i + '_rcount'] = f_ratio[i]\n",
    "#     f_ratio[i + '_ratio'] = f_ratio[i]/len(train)\n",
    "#     f_ratio[i + '_ratio'] = f_ratio[i + '_ratio'].astype('float32')\n",
    "#     f_ratio[i] = f_ratio.index\n",
    "#     f_ratio = f_ratio.reset_index(drop = True)\n",
    "#     train = train.merge(f_ratio,on = i,how = 'left')\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_corr = train.corr()\n",
    "\n",
    "# # Set the threshold to select only highly correlated attributes\n",
    "# threshold = 0.9\n",
    "\n",
    "# # List of pairs along with correlation above threshold\n",
    "# corr_list = []\n",
    "# size = len(train.columns)\n",
    "# #Search for the highly correlated pairs\n",
    "# for i in range(0,size): #for 'size' features\n",
    "#     for j in range(i+1,size): #avoid repetition\n",
    "#         if (data_corr.iloc[i,j] >= threshold and data_corr.iloc[i,j] < 1) or (data_corr.iloc[i,j] < 0 and data_corr.iloc[i,j] <= -threshold):\n",
    "#             corr_list.append([data_corr.iloc[i,j],i,j]) #store correlation and columns index\n",
    "\n",
    "# #Sort to show higher ones first            \n",
    "# s_corr_list = sorted(corr_list,key=lambda x: -abs(x[0]))\n",
    "\n",
    "# #Print correlations and column names\n",
    "# # for v,i,j in s_corr_list:\n",
    "# #     print (\"%s and %s = %.2f\" % (train.columns[i],train.columns[j],v))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Filling Nan!\")\n",
    "# train = train.replace(-1,np.nan)\n",
    "# train.isnull().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Filling Nan by LGB!\")\n",
    "# predict_features = []\n",
    "# for i in train.columns:\n",
    "#     if(train[i].isnull().sum() != 0):\n",
    "#         #if(float(train[i].isnull().sum()) / len(train) < 0.2):\n",
    "#         predict_features.append(i)\n",
    "# train = set_missing(train,predict_features)\n",
    "\n",
    "# train.isnull().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.fillna(-1)\n",
    "train.isnull().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Getting last trading time delta!\")\n",
    "# train = train.sort_values(['date'])\n",
    "# train = train.reset_index(drop=True)\n",
    "# train['year'] = train['date'].values / 10000\n",
    "# train['month'] = train['date'].values /100 % 100\n",
    "# train['day'] = train['date'].values % 100\n",
    "# train['convertDays'] = train['year'].values * 365 + (train['month'].values - 1)* 30 + train['day'].values\n",
    "# train['time_delta'] = train.sort_values(['date']).groupby(by=userid)[['convertDays']].diff()\n",
    "# train = train.drop(['month','day','year','convertDays'],axis=1)\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Getting trade times!\")\n",
    "# gp = train[userid + ['date']].groupby(by=userid)['date'].count().reset_index().rename(columns={'date': 'trade_count'})\n",
    "# print(\"Merging!\")\n",
    "# train = train.merge(gp, on=userid, how='left')\n",
    "# train['trade_count'] = train['trade_count'].astype('uint16')\n",
    "# del gp\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Getting trade unique times!\")\n",
    "# gp = train[userid + ['date']].groupby(by=userid)['date'].nunique().reset_index().rename(columns={'date': 'trade_unique'})\n",
    "# print(\"Merging!\")\n",
    "# train = train.merge(gp, on=userid, how='left')\n",
    "# train['trade_unique'] = train['trade_unique'].astype('uint16')\n",
    "# del gp\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Getting trade cumcount times!\")\n",
    "# train = train.sort_values(['date'])\n",
    "\n",
    "# train = train.reset_index(drop = True)\n",
    "# train['trade_cumcount'] = train[userid + ['date']].groupby(by=userid)['date'].cumcount() + 1\n",
    "# train['trade_cumcount'] = train['trade_cumcount'].astype('uint16')\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Getting first trade time\")\n",
    "# train_ = train[userid + ['date','trade_cumcount']]\n",
    "# train_ = train_.sort_values(by=['trade_cumcount']).reset_index(drop = True)\n",
    "# train_ = train_[:(np.where(train_['trade_cumcount'] ==2)[0][0])]\n",
    "# train_['first_trade_time'] = train_['date'] * train_['trade_cumcount']\n",
    "# train_ = train_.drop(['date','trade_cumcount'],axis=1) \n",
    "# train = train.merge(train_,on=userid, how='left')\n",
    "# del train_\n",
    "# train['trade_cumcount'] = train['trade_cumcount'] - 1\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Get important features!\")\n",
    "# group1 = ['f204','f205','f206','f207']\n",
    "# group2 = ['f208','f209','f210']\n",
    "# group3 = ['f215','f216','f217','f218']\n",
    "# group4 = ['f234','f235','f236','f237','f238']\n",
    "# group5 = ['f243','f244','f245','f246','f247','f248','f253']\n",
    "# group6 = ['f259','f260','f261','f262','f263','f266']\n",
    "# groups = [group1,group2,group3,group4,group5,group6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Doing interaction between important features!\")\n",
    "# for i in groups:\n",
    "#     print(\"Dealing with\",i)\n",
    "#     for e, (x, y) in enumerate(combinations(i, 2)):\n",
    "#         interaction_features(train,x,y,e)\n",
    "#     print(train.shape)\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Getting proj features!\")\n",
    "# for i in group1:\n",
    "#     print(\"Dealing with\",i,\"and f7\")\n",
    "#     df = proj_num_on_cat(train,i,'f7')\n",
    "#     for i in df.columns:\n",
    "#         df[i] = df[i].astype('uint32')\n",
    "#     train = pd.concat([train,df],axis = 1)\n",
    "#     print(\"train shape\",train.shape)\n",
    "# del df\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"One_Hot_Encoder!\")\n",
    "# ohe = []\n",
    "# #need to change to 1-19\n",
    "# col = userid + ['f1','f2','f3','f4']\n",
    "# for column in col:\n",
    "#     temp = pd.get_dummies(pd.Series(train[column]))\n",
    "#     for i in temp.columns:\n",
    "#         temp.rename(columns = {i : column + '_' + str(i)}, inplace = True)\n",
    "#         ohe.append(column + '_' + str(i))\n",
    "#     train = pd.concat([train,temp],axis=1)\n",
    "#     train = train.drop([column],axis=1)\n",
    "\n",
    "# print(len(ohe))\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Counting cat features!\")\n",
    "# col = userid + ['f1','f2','f3','f4']\n",
    "# cat_count_df = cat_count(train, col)\n",
    "# for i in cat_count_df.columns:\n",
    "#     cat_count_df[i] = cat_count_df[i].astype('uint32')\n",
    "# train = pd.concat([train,cat_count_df],axis=1)\n",
    "# del cat_count_df\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1495269 entries, 0 to 1495268\n",
      "Columns: 176 entries, date to nan_percent\n",
      "dtypes: float32(153), int64(1), uint16(4), uint8(18)\n",
      "memory usage: 932.6 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Starting subsample!\")\n",
    "# test = train[len_train:]\n",
    "# train = train[:len_train]\n",
    "# print(\"train size\",len(train))\n",
    "# print(\"test size\",len(test))\n",
    "# # target = pd.DataFrame(target)\n",
    "# # train = pd.concat([train,target],axis=1)\n",
    "# print(train.shape,test.shape)\n",
    "# print(\"Split into 0 and 1!\")\n",
    "# train0 = train[train['label'] == 0]\n",
    "# train1 = train[train['label'] == 1]\n",
    "# train0_X,val0_X, train0_y, val0_y = train_test_split(train0,train0['label'].values,test_size = int(len(train) *5),random_state = 0) \n",
    "# df = pd.DataFrame(val0_X,columns=train.columns)\n",
    "# print(\"DF with 0 label shape\", df.shape)\n",
    "# train_ = df.append(train1)\n",
    "# train_ = train.reset_index(drop=True)\n",
    "# print(\"Preparing data for training!\")\n",
    "# target = train_['label'].values\n",
    "# train_ = train_.drop(['label'],axis=1)\n",
    "# X = train_.values\n",
    "# y = target\n",
    "# print(\"X\",X.shape,\"Y\",y.shape)\n",
    "# sub = pd.read_csv('atec_anti_fraud_test_a.csv', usecols=['id'])\n",
    "# sub['score']=0\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Dealing with -1 label!\")\n",
    "# train = train.drop(['date'],axis = 1)\n",
    "# test = train[len_train:]\n",
    "# train = train[:len_train]\n",
    "# test = test.drop(['label'],axis = 1)\n",
    "# df = train[train['label'] == -1]\n",
    "\n",
    "# print(\"Delete -1 in train!\")\n",
    "# train.drop(np.where(train.label == -1)[0],inplace=True)\n",
    "# train = train.reset_index(drop = True)\n",
    "# df = df.reset_index(drop = True)\n",
    "\n",
    "# print(\"Drop label!\")\n",
    "# df = df.drop(['label'],axis=1)\n",
    "# label = train['label'].values\n",
    "# train = train.drop(['label'],axis=1)\n",
    "\n",
    "# print(\"train shape\",train.shape)\n",
    "# print(\"test shape\",test.shape)\n",
    "# print(\"df shape\",df.shape)\n",
    "\n",
    "# train_X,val_X, train_y, val_y = train_test_split(train,label,test_size = 0.25,random_state = 0) \n",
    "\n",
    "# print(\"Preparing model!\")\n",
    "# clf = lgb.LGBMClassifier(\n",
    "#         boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "#         max_depth=-1, n_estimators=3000, objective='binary',\n",
    "#         subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "#         learning_rate=0.05, min_child_weight=50, random_state=2018, n_jobs=100\n",
    "#     )\n",
    "# clf.fit(train_X.values,train_y,eval_set=[(val_X.values, val_y)],eval_metric='auc',early_stopping_rounds=30,verbose=100)\n",
    "# result = clf.predict(df.values,num_iteration=clf.best_iteration_)\n",
    "\n",
    "# print(\"Preparing data for training!\")\n",
    "# df['label'] = result\n",
    "# train['label'] = label\n",
    "# train = train.append(df)\n",
    "# train = train.reset_index(drop=True)\n",
    "# label = train['label']\n",
    "# train = train.drop(['label'],axis=1)\n",
    "\n",
    "# X = train.values\n",
    "# y= label.values\n",
    "# print(\"X\",X.shape,\"Y\",y.shape)\n",
    "# print(\"Train size\",train.shape)\n",
    "# print(\"Test size\",test.shape)\n",
    "\n",
    "# print(\"Preparing sub!\")\n",
    "# sub = pd.read_csv('atec_anti_fraud_test_a.csv', usecols=['id'])\n",
    "# sub['score']=0\n",
    "\n",
    "# del train\n",
    "# gc.collect()\n",
    "# print(\"All done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing data for training!\n",
      "(1495269, 175)\n",
      "('X', (994731, 175), 'Y', (994731,))\n",
      "('Train size', (994731, 175))\n",
      "('Test size', (500538, 175))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Preparing data for training!\")\n",
    "train = train.drop(['date'],axis = 1)\n",
    "#train = train[:len_test]\n",
    "print train.shape\n",
    "test = train[len_train:]\n",
    "train = train[:len_train]\n",
    "y = label.values\n",
    "#train = train.drop(['label'],axis=1)\n",
    "X = train.values\n",
    "print(\"X\",X.shape,\"Y\",y.shape)\n",
    "print(\"Train size\",train.shape)\n",
    "print(\"Test size\",test.shape)\n",
    "sub = pd.read_csv('atec_anti_fraud_test_a.csv', usecols=['id'])\n",
    "sub['score']=0\n",
    "del train\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Creating XGB features!\")\n",
    "# print(\"Current X\",X.shape,\"test\",test.shape)\n",
    "# [X,y,test] = xb.fit_model(X,y,test.values)\n",
    "# print(\"After Current X\",X.shape,\"test\",test.shape)\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"XGB!\")\n",
    "# xgbparams = {'eta': 0.1, \n",
    "#           'tree_method': \"auto\", \n",
    "#           'max_depth': 8, \n",
    "#           'subsample': 0.7, \n",
    "#           'colsample_bytree': 0.7, \n",
    "#           'colsample_bylevel':0.1,\n",
    "#           'min_child_weight':16,\n",
    "#           #'alpha':4,\n",
    "#           'objective': 'binary:logistic', \n",
    "#           'scale_pos_weight':99,\n",
    "#           'eval_metric': 'auc', \n",
    "#           'random_state': 99,\n",
    "#           'silent': True}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"XGB!\")\n",
    "# xgbparams = {'eta': 0.1, \n",
    "#           'tree_method': \"auto\", \n",
    "#           'max_depth': 4, \n",
    "#           'subsample': 0.7, \n",
    "#           'colsample_bytree': 0.8, \n",
    "#           'colsample_bylevel':0.8,\n",
    "#           'min_child_weight':10,\n",
    "#           'gamma':0.6,   \n",
    "#           #'reg_alpha':1e-05,\n",
    "#           #'reg_lambda':1,\n",
    "#           'objective': 'binary:logistic', \n",
    "#           'scale_pos_weight':99,\n",
    "#           'eval_metric': 'auc', \n",
    "#           'random_state': 99,\n",
    "#           'silent': True}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_X,val_X, train_y, val_y = train_test_split(train,target,test_size = 0.1,random_state = 0) \n",
    "# train_y = train_y.astype('uint8')\n",
    "# val_y = val_y.astype('uint8')\n",
    "# print('The size of the test set is ', len(test))\n",
    "# print('The size of the validation set is ', len(val_X))\n",
    "# print('The size of the train set is ', len(train_X))\n",
    "# dtrain = xgb.DMatrix(train_X, train_y)\n",
    "# del train_X, train_y\n",
    "# dvalid = xgb.DMatrix(val_X, val_y)\n",
    "# del val_X, val_y\n",
    "# watchlist = [(dtrain, 'train'), (dvalid, 'valid')]\n",
    "# model = xgb.train(xgbparams, dtrain, 1000, watchlist, maximize=True, early_stopping_rounds=20, verbose_eval=10)\n",
    "# del dtrain\n",
    "# del dvalid\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def ceate_feature_map(features):  \n",
    "#     outfile = open('xgb.fmap', 'w')  \n",
    "#     i = 0  \n",
    "#     for feat in features:  \n",
    "#         outfile.write('{0}\\t{1}\\tq\\n'.format(i, feat))  \n",
    "#         i = i + 1  \n",
    "#     outfile.close() \n",
    "    \n",
    "# print(\"saveing feat_importance!\")\n",
    "# features = [x for x in train.columns if x not in ['id','loss']]  \n",
    "# ceate_feature_map(features)  \n",
    "  \n",
    "# importance = xgb_model.get_fscore(fmap='xgb.fmap')  \n",
    "# importance = sorted(importance.items(), key=operator.itemgetter(1))  \n",
    "  \n",
    "# df = pd.DataFrame(importance, columns=['feature', 'fscore'])  \n",
    "# df['fscore1'] = df['fscore'] / df['fscore'].sum()  \n",
    "# df.to_csv(\"feat_importance.csv\", index=False)  \n",
    "\n",
    "# plt.figure()  \n",
    "# df.plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(50, 50))  \n",
    "# plt.title('XGBoost Feature Importance1')  \n",
    "# plt.xlabel('relative importance')  \n",
    "# plt.savefig(\"feat_importance.jpg\",dpi = 400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = X[0:length]\n",
    "# X_valid = X[length:]\n",
    "# y_train = y[0:length]\n",
    "# y_valid = y[length:]\n",
    "d_train = xgb.DMatrix(X_train, y_train) \n",
    "d_valid = xgb.DMatrix(X_valid, y_valid)\n",
    "watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "xgb_model = xgb.train(xgbparams, d_train, 5000, watchlist, early_stopping_rounds=100,maximize=True, verbose_eval=100)\n",
    "# sub['score'] = xgb_model.predict(xgb.DMatrix(test.values), ntree_limit=xgb_model.best_ntree_limit +150)\n",
    "\n",
    "# sub.to_csv('sub_sinxgb_xgbfit_nt_params.csv',index=False)\n",
    "# print(\"Is this Done? No, he fucked up!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbparams = {'eta': 0.1, \n",
    "          'tree_method': \"auto\", \n",
    "          'max_depth': 4, \n",
    "          'subsample': 0.7, \n",
    "          'colsample_bytree': 0.1, \n",
    "          'colsample_bylevel':0.1,\n",
    "          'min_child_weight':10,\n",
    "          'gamma':0.6,   \n",
    "          #'reg_alpha':1e-05,\n",
    "          #'reg_lambda':1,\n",
    "          'objective': 'binary:logistic', \n",
    "          'scale_pos_weight':1,\n",
    "          'eval_metric': 'auc', \n",
    "          'random_state': 99,\n",
    "          'silent': True}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-fold training!\n",
      "('train index', array([198937, 198938, 198939, ..., 994728, 994729, 994730]))\n",
      " xgb kfold: 1  of  5 : \n",
      "[0]\ttrain-auc:0.664066\tvalid-auc:0.657432\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 70 rounds.\n",
      "[100]\ttrain-auc:0.955802\tvalid-auc:0.956845\n",
      "[200]\ttrain-auc:0.965596\tvalid-auc:0.966013\n",
      "[300]\ttrain-auc:0.97111\tvalid-auc:0.970631\n",
      "[400]\ttrain-auc:0.973592\tvalid-auc:0.972529\n",
      "[500]\ttrain-auc:0.975135\tvalid-auc:0.973748\n",
      "[600]\ttrain-auc:0.976169\tvalid-auc:0.97451\n",
      "[700]\ttrain-auc:0.977058\tvalid-auc:0.975087\n",
      "[800]\ttrain-auc:0.97773\tvalid-auc:0.975588\n",
      "[900]\ttrain-auc:0.97834\tvalid-auc:0.975963\n",
      "[1000]\ttrain-auc:0.978714\tvalid-auc:0.976164\n",
      "[1100]\ttrain-auc:0.979148\tvalid-auc:0.9764\n",
      "[1200]\ttrain-auc:0.97953\tvalid-auc:0.976612\n",
      "[1300]\ttrain-auc:0.979889\tvalid-auc:0.97679\n",
      "[1400]\ttrain-auc:0.980211\tvalid-auc:0.977009\n",
      "[1500]\ttrain-auc:0.980494\tvalid-auc:0.977134\n",
      "[1600]\ttrain-auc:0.980774\tvalid-auc:0.977239\n",
      "[1700]\ttrain-auc:0.981078\tvalid-auc:0.977403\n",
      "[1800]\ttrain-auc:0.981358\tvalid-auc:0.977632\n",
      "[1900]\ttrain-auc:0.981588\tvalid-auc:0.97775\n",
      "[2000]\ttrain-auc:0.981833\tvalid-auc:0.977838\n",
      "[2100]\ttrain-auc:0.98204\tvalid-auc:0.977903\n",
      "[2200]\ttrain-auc:0.98234\tvalid-auc:0.978098\n",
      "[2300]\ttrain-auc:0.982529\tvalid-auc:0.978207\n",
      "[2400]\ttrain-auc:0.982682\tvalid-auc:0.978251\n",
      "[2500]\ttrain-auc:0.982916\tvalid-auc:0.978371\n",
      "[2600]\ttrain-auc:0.983127\tvalid-auc:0.978485\n",
      "[2700]\ttrain-auc:0.983325\tvalid-auc:0.978531\n",
      "[2800]\ttrain-auc:0.983476\tvalid-auc:0.978567\n",
      "[2900]\ttrain-auc:0.983666\tvalid-auc:0.97862\n",
      "[3000]\ttrain-auc:0.983819\tvalid-auc:0.978738\n",
      "[3100]\ttrain-auc:0.984007\tvalid-auc:0.978827\n",
      "[3200]\ttrain-auc:0.984163\tvalid-auc:0.978899\n",
      "[3300]\ttrain-auc:0.98432\tvalid-auc:0.978995\n",
      "[3400]\ttrain-auc:0.984484\tvalid-auc:0.979072\n",
      "[3500]\ttrain-auc:0.984667\tvalid-auc:0.979099\n",
      "[3600]\ttrain-auc:0.984788\tvalid-auc:0.979111\n",
      "Stopping. Best iteration:\n",
      "[3538]\ttrain-auc:0.984697\tvalid-auc:0.979144\n",
      "\n",
      "('train index', array([     0,      1,      2, ..., 994728, 994729, 994730]))\n",
      " xgb kfold: 2  of  5 : \n",
      "[0]\ttrain-auc:0.662996\tvalid-auc:0.661712\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 70 rounds.\n",
      "[100]\ttrain-auc:0.957265\tvalid-auc:0.955464\n",
      "[200]\ttrain-auc:0.966289\tvalid-auc:0.964654\n",
      "[300]\ttrain-auc:0.971234\tvalid-auc:0.969591\n",
      "[400]\ttrain-auc:0.973649\tvalid-auc:0.972073\n",
      "[500]\ttrain-auc:0.975133\tvalid-auc:0.973294\n",
      "[600]\ttrain-auc:0.976256\tvalid-auc:0.974199\n",
      "[700]\ttrain-auc:0.977033\tvalid-auc:0.974797\n",
      "[800]\ttrain-auc:0.97769\tvalid-auc:0.975242\n",
      "[900]\ttrain-auc:0.978406\tvalid-auc:0.975755\n",
      "[1000]\ttrain-auc:0.978996\tvalid-auc:0.976151\n",
      "[1100]\ttrain-auc:0.979355\tvalid-auc:0.976387\n",
      "[1200]\ttrain-auc:0.979723\tvalid-auc:0.976648\n",
      "[1300]\ttrain-auc:0.980083\tvalid-auc:0.97685\n",
      "[1400]\ttrain-auc:0.980402\tvalid-auc:0.977064\n",
      "[1500]\ttrain-auc:0.980663\tvalid-auc:0.977194\n",
      "[1600]\ttrain-auc:0.980957\tvalid-auc:0.977334\n",
      "[1700]\ttrain-auc:0.981226\tvalid-auc:0.977592\n",
      "[1800]\ttrain-auc:0.98149\tvalid-auc:0.977809\n",
      "[1900]\ttrain-auc:0.981748\tvalid-auc:0.977924\n",
      "[2000]\ttrain-auc:0.981967\tvalid-auc:0.978063\n",
      "[2100]\ttrain-auc:0.982158\tvalid-auc:0.978183\n",
      "[2200]\ttrain-auc:0.982399\tvalid-auc:0.978375\n",
      "[2300]\ttrain-auc:0.982599\tvalid-auc:0.978495\n",
      "[2400]\ttrain-auc:0.982757\tvalid-auc:0.97853\n",
      "[2500]\ttrain-auc:0.982985\tvalid-auc:0.978679\n",
      "[2600]\ttrain-auc:0.983177\tvalid-auc:0.978766\n",
      "[2700]\ttrain-auc:0.983383\tvalid-auc:0.978825\n",
      "Stopping. Best iteration:\n",
      "[2661]\ttrain-auc:0.983321\tvalid-auc:0.978846\n",
      "\n",
      "('train index', array([     0,      1,      2, ..., 994728, 994729, 994730]))\n",
      " xgb kfold: 3  of  5 : \n",
      "[0]\ttrain-auc:0.684751\tvalid-auc:0.68711\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 70 rounds.\n",
      "[100]\ttrain-auc:0.954836\tvalid-auc:0.95129\n",
      "[200]\ttrain-auc:0.965812\tvalid-auc:0.962149\n",
      "[300]\ttrain-auc:0.971241\tvalid-auc:0.967577\n",
      "[400]\ttrain-auc:0.973876\tvalid-auc:0.970084\n",
      "[500]\ttrain-auc:0.975205\tvalid-auc:0.971211\n",
      "[600]\ttrain-auc:0.976474\tvalid-auc:0.972468\n",
      "[700]\ttrain-auc:0.977498\tvalid-auc:0.973346\n",
      "[800]\ttrain-auc:0.978383\tvalid-auc:0.974156\n",
      "[900]\ttrain-auc:0.978845\tvalid-auc:0.974516\n",
      "[1000]\ttrain-auc:0.979384\tvalid-auc:0.97492\n",
      "[1100]\ttrain-auc:0.979756\tvalid-auc:0.975127\n",
      "[1200]\ttrain-auc:0.980069\tvalid-auc:0.975317\n",
      "[1300]\ttrain-auc:0.980378\tvalid-auc:0.97551\n",
      "[1400]\ttrain-auc:0.980692\tvalid-auc:0.975654\n",
      "[1500]\ttrain-auc:0.980979\tvalid-auc:0.975825\n",
      "[1600]\ttrain-auc:0.981233\tvalid-auc:0.975996\n",
      "[1700]\ttrain-auc:0.981488\tvalid-auc:0.976102\n",
      "[1800]\ttrain-auc:0.981719\tvalid-auc:0.976192\n",
      "[1900]\ttrain-auc:0.981981\tvalid-auc:0.97633\n",
      "[2000]\ttrain-auc:0.982227\tvalid-auc:0.976472\n",
      "[2100]\ttrain-auc:0.982459\tvalid-auc:0.976583\n",
      "[2200]\ttrain-auc:0.982664\tvalid-auc:0.976692\n",
      "[2300]\ttrain-auc:0.982918\tvalid-auc:0.976784\n",
      "[2400]\ttrain-auc:0.983122\tvalid-auc:0.976927\n",
      "[2500]\ttrain-auc:0.983344\tvalid-auc:0.977058\n",
      "[2600]\ttrain-auc:0.983491\tvalid-auc:0.977107\n",
      "[2700]\ttrain-auc:0.983704\tvalid-auc:0.977184\n",
      "[2800]\ttrain-auc:0.983862\tvalid-auc:0.977242\n",
      "[2900]\ttrain-auc:0.984006\tvalid-auc:0.977301\n",
      "[3000]\ttrain-auc:0.98417\tvalid-auc:0.977356\n",
      "[3100]\ttrain-auc:0.984335\tvalid-auc:0.977489\n",
      "[3200]\ttrain-auc:0.984469\tvalid-auc:0.977534\n",
      "[3300]\ttrain-auc:0.984644\tvalid-auc:0.977619\n",
      "[3400]\ttrain-auc:0.984812\tvalid-auc:0.977718\n",
      "[3500]\ttrain-auc:0.984978\tvalid-auc:0.977775\n",
      "[3600]\ttrain-auc:0.985169\tvalid-auc:0.97791\n",
      "[3700]\ttrain-auc:0.985289\tvalid-auc:0.977961\n",
      "[3800]\ttrain-auc:0.985407\tvalid-auc:0.977992\n",
      "[3900]\ttrain-auc:0.98555\tvalid-auc:0.978085\n",
      "[4000]\ttrain-auc:0.985689\tvalid-auc:0.978116\n",
      "[4100]\ttrain-auc:0.98582\tvalid-auc:0.978169\n",
      "[4200]\ttrain-auc:0.985943\tvalid-auc:0.978215\n",
      "[4300]\ttrain-auc:0.986085\tvalid-auc:0.978278\n",
      "[4400]\ttrain-auc:0.986225\tvalid-auc:0.97834\n",
      "Stopping. Best iteration:\n",
      "[4388]\ttrain-auc:0.986212\tvalid-auc:0.978353\n",
      "\n",
      "('train index', array([     0,      1,      2, ..., 994728, 994729, 994730]))\n",
      " xgb kfold: 4  of  5 : \n",
      "[0]\ttrain-auc:0.680389\tvalid-auc:0.679521\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 70 rounds.\n",
      "[100]\ttrain-auc:0.953838\tvalid-auc:0.952809\n",
      "[200]\ttrain-auc:0.965774\tvalid-auc:0.96511\n",
      "[300]\ttrain-auc:0.97068\tvalid-auc:0.96988\n",
      "[400]\ttrain-auc:0.973232\tvalid-auc:0.972444\n",
      "[500]\ttrain-auc:0.974489\tvalid-auc:0.973455\n",
      "[600]\ttrain-auc:0.97577\tvalid-auc:0.974553\n",
      "[700]\ttrain-auc:0.976723\tvalid-auc:0.975377\n",
      "[800]\ttrain-auc:0.977565\tvalid-auc:0.976006\n",
      "[900]\ttrain-auc:0.978059\tvalid-auc:0.976184\n",
      "[1000]\ttrain-auc:0.978597\tvalid-auc:0.976647\n",
      "[1100]\ttrain-auc:0.978978\tvalid-auc:0.976824\n",
      "[1200]\ttrain-auc:0.979295\tvalid-auc:0.977017\n",
      "[1300]\ttrain-auc:0.979618\tvalid-auc:0.977191\n",
      "[1400]\ttrain-auc:0.97996\tvalid-auc:0.977375\n",
      "[1500]\ttrain-auc:0.980272\tvalid-auc:0.977621\n",
      "[1600]\ttrain-auc:0.98057\tvalid-auc:0.977747\n",
      "[1700]\ttrain-auc:0.980864\tvalid-auc:0.977886\n",
      "[1800]\ttrain-auc:0.981088\tvalid-auc:0.977984\n",
      "[1900]\ttrain-auc:0.981327\tvalid-auc:0.978073\n",
      "[2000]\ttrain-auc:0.981555\tvalid-auc:0.978197\n",
      "[2100]\ttrain-auc:0.981813\tvalid-auc:0.978319\n",
      "[2200]\ttrain-auc:0.982031\tvalid-auc:0.978354\n",
      "[2300]\ttrain-auc:0.982272\tvalid-auc:0.978516\n",
      "[2400]\ttrain-auc:0.982495\tvalid-auc:0.978641\n",
      "[2500]\ttrain-auc:0.982742\tvalid-auc:0.978799\n",
      "[2600]\ttrain-auc:0.982916\tvalid-auc:0.978873\n",
      "[2700]\ttrain-auc:0.983084\tvalid-auc:0.978893\n",
      "[2800]\ttrain-auc:0.983278\tvalid-auc:0.978956\n",
      "[2900]\ttrain-auc:0.983445\tvalid-auc:0.978992\n",
      "[3000]\ttrain-auc:0.983639\tvalid-auc:0.979068\n",
      "[3100]\ttrain-auc:0.983801\tvalid-auc:0.979103\n",
      "Stopping. Best iteration:\n",
      "[3064]\ttrain-auc:0.983758\tvalid-auc:0.979132\n",
      "\n",
      "('train index', array([     0,      1,      2, ..., 795796, 795797, 795798]))\n",
      " xgb kfold: 5  of  5 : \n",
      "[0]\ttrain-auc:0.696867\tvalid-auc:0.707867\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 70 rounds.\n",
      "[100]\ttrain-auc:0.953753\tvalid-auc:0.954564\n",
      "[200]\ttrain-auc:0.966054\tvalid-auc:0.96579\n",
      "[300]\ttrain-auc:0.969888\tvalid-auc:0.969019\n",
      "[400]\ttrain-auc:0.972884\tvalid-auc:0.971566\n",
      "[500]\ttrain-auc:0.974326\tvalid-auc:0.972658\n",
      "[600]\ttrain-auc:0.975601\tvalid-auc:0.973803\n",
      "[700]\ttrain-auc:0.976212\tvalid-auc:0.974238\n",
      "[800]\ttrain-auc:0.977077\tvalid-auc:0.974912\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[900]\ttrain-auc:0.977695\tvalid-auc:0.975353\n",
      "[1000]\ttrain-auc:0.978545\tvalid-auc:0.976045\n",
      "[1100]\ttrain-auc:0.979189\tvalid-auc:0.97656\n",
      "[1200]\ttrain-auc:0.97957\tvalid-auc:0.976726\n",
      "[1300]\ttrain-auc:0.979897\tvalid-auc:0.976955\n",
      "[1400]\ttrain-auc:0.980314\tvalid-auc:0.97726\n",
      "[1500]\ttrain-auc:0.980631\tvalid-auc:0.977426\n",
      "[1600]\ttrain-auc:0.980923\tvalid-auc:0.977539\n",
      "[1700]\ttrain-auc:0.9812\tvalid-auc:0.977728\n",
      "[1800]\ttrain-auc:0.98149\tvalid-auc:0.977844\n",
      "[1900]\ttrain-auc:0.981733\tvalid-auc:0.977984\n",
      "[2000]\ttrain-auc:0.981983\tvalid-auc:0.978162\n",
      "[2100]\ttrain-auc:0.982216\tvalid-auc:0.978226\n",
      "[2200]\ttrain-auc:0.982413\tvalid-auc:0.978326\n",
      "[2300]\ttrain-auc:0.982653\tvalid-auc:0.978498\n",
      "[2400]\ttrain-auc:0.982858\tvalid-auc:0.978587\n",
      "[2500]\ttrain-auc:0.983062\tvalid-auc:0.978698\n",
      "Stopping. Best iteration:\n",
      "[2506]\ttrain-auc:0.983067\tvalid-auc:0.978709\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"K-fold training!\")\n",
    "sub = pd.read_csv('atec_anti_fraud_test_b.csv', usecols=['id'])\n",
    "sub['score']=0\n",
    "#del train\n",
    "#stacking = train.f1\n",
    "#print(\"stacking length\",len(stacking))\n",
    "#result = np.zeros(len(test))\n",
    "#print(\"stacking result length\",len(result))\n",
    "bst_xgb = []\n",
    "nrounds=5000  \n",
    "kfold = 5  \n",
    "skf = StratifiedKFold(n_splits=kfold, random_state=0)\n",
    "for i, (train_index, test_index) in enumerate(skf.split(X, y)):\n",
    "    print(\"train index\",train_index)\n",
    "    print(' xgb kfold: {}  of  {} : '.format(i+1, kfold))\n",
    "    X_train, X_valid = X[train_index], X[test_index]\n",
    "    y_train, y_valid = y[train_index], y[test_index]\n",
    "    d_train = xgb.DMatrix(X_train, y_train) \n",
    "    d_valid = xgb.DMatrix(X_valid, y_valid) \n",
    "    watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "    xgb_model = xgb.train(xgbparams, d_train, nrounds, watchlist, early_stopping_rounds=70,maximize=True, verbose_eval=100)\n",
    "    sub['score'] += xgb_model.predict(xgb.DMatrix(test.values), ntree_limit= xgb_model.best_ntree_limit+160) / (2 * kfold)\n",
    "    #result += xgb_model.predict(xgb.DMatrix(test), ntree_limit=xgb_model.best_ntree_limit +150) / (kfold)\n",
    "    #bst_xgb.append(xgb_model.best_ntree_limit)\n",
    "    #stacking[test_index] = xgb_model.predict(d_valid, ntree_limit=xgb_model.best_ntree_limit +150)\n",
    "\n",
    "    \n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,5):\n",
    "    print i\n",
    "    sub['score'] += xgb_model.predict(xgb.DMatrix(test.values), ntree_limit=xgb_model.best_ntree_limit+160) / (2*kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is this Done? No, he fucked up!\n"
     ]
    }
   ],
   "source": [
    "sub.to_csv('sub_xgb_40b_1.csv',index=False)\n",
    "print(\"Is this Done? No, he fucked up!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Stacking!\")\n",
    "# test = convertToOne(test,result)\n",
    "# X = convertToOne(X,stacking.values)\n",
    "# print(\"After Current X\",X.shape,\"test\",test.shape)\n",
    "# print(\"Training!\")\n",
    "# xgb_model = xgb.train(xgbparams,xgb.DMatrix(train.values, y), int(np.sum(bst_xgb) / 5 *1.1), verbose_eval=50)\n",
    "# print(\"Predicting!\")\n",
    "# sub = pd.read_csv('atec_anti_fraud_test_a.csv', usecols=['id'])\n",
    "# sub['score']=0\n",
    "# sub['score'] = xgb_model.predict(xgb.DMatrix(test.values))\n",
    "# sub.to_csv('sub_xgb_me_nt_stacking.csv',index=False)\n",
    "# print(\"Is this Done? No, he fucked up!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lgbparams = {\n",
    "#     'boosting_type': 'gbdt',\n",
    "#     'objective': 'binary',\n",
    "#     'metric': 'auc',\n",
    "#     'learning_rate': 0.1,\n",
    "#     'num_leaves': 31,  \n",
    "#     'max_depth': -1,  \n",
    "#     'min_child_samples': 100,  \n",
    "#     'max_bin': 100,  \n",
    "#     'subsample': 0.7,  \n",
    "#     'subsample_freq': 1,  \n",
    "#     'colsample_bytree': 0.7,  \n",
    "#     'min_child_weight': 10,  \n",
    "#     #'subsample_for_bin': 200000,  \n",
    "#     'min_split_gain': 0,  \n",
    "#     'reg_alpha': 0,  \n",
    "#     'reg_lambda': 0,  \n",
    "#    # 'nthread': 8,\n",
    "#     'verbose': 0,\n",
    "#     'scale_pos_weight':100 \n",
    "#     }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sub['score']=0\n",
    "# print(\"K-fold lgb training!\")\n",
    "# kfold = 5   \n",
    "# nrounds = 2000\n",
    "# skf = StratifiedKFold(n_splits=kfold, random_state=0)\n",
    "# for i, (train_index, test_index) in enumerate(skf.split(X, y)):\n",
    "#     print(' xgb kfold: {}  of  {} : '.format(i+1, kfold))\n",
    "#     X_train, X_valid = X[train_index], X[test_index]\n",
    "#     y_train, y_valid = y[train_index], y[test_index]\n",
    "#     lgb_model = lgb.train(lgbparams, lgb.Dataset(X_train, label=y_train), nrounds, \n",
    "#                   lgb.Dataset(X_valid, label=y_valid), verbose_eval=100, early_stopping_rounds=30)\n",
    "#     sub['score'] += lgb_model.predict(test.values,num_iteration=lgb_model.best_iteration) / (kfold)\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sub.to_csv('sub_lgb_me_-1_notime.csv',index=False)\n",
    "# print(\"Is this Done? No, he fucked up!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "## print(\"Tuning!\")\n",
    "# # tuning\n",
    "# ITERATIONS = 1000 # 1000\n",
    "# # Classifier\n",
    "# bayes_cv_tuner = BayesSearchCV(\n",
    "#     estimator = xgb.XGBClassifier(\n",
    "#         n_jobs = 1,\n",
    "#         objective = 'binary:logistic',\n",
    "#         eval_metric = 'auc',\n",
    "#         silent=1,\n",
    "#         tree_method='approx'\n",
    "#     ),\n",
    "#     search_spaces = {\n",
    "#         'learning_rate': (0.01, 1.0, 'log-uniform'),\n",
    "#         'min_child_weight': (0, 10),\n",
    "#         'max_depth': (0, 50),\n",
    "#         'max_delta_step': (0, 20),\n",
    "#         'subsample': (0.01, 1.0, 'uniform'),\n",
    "#         'colsample_bytree': (0.01, 1.0, 'uniform'),\n",
    "#         'colsample_bylevel': (0.01, 1.0, 'uniform'),\n",
    "#         'reg_lambda': (1e-9, 1000, 'log-uniform'),\n",
    "#         'reg_alpha': (1e-9, 1.0, 'log-uniform'),\n",
    "#         'gamma': (1e-9, 0.5, 'log-uniform'),\n",
    "#         'min_child_weight': (0, 5),\n",
    "#         'n_estimators': (50, 100),\n",
    "#         'scale_pos_weight': (1e-6, 500, 'log-uniform')\n",
    "#     },    \n",
    "#     scoring = 'roc_auc',\n",
    "#     cv = StratifiedKFold(\n",
    "#         n_splits=3,\n",
    "#         shuffle=True,\n",
    "#         random_state=42\n",
    "#     ),\n",
    "#     n_jobs = 3,\n",
    "#     n_iter = ITERATIONS,   \n",
    "#     verbose = 0,\n",
    "#     refit = True,\n",
    "#     random_state = 42\n",
    "# )\n",
    "# def status_print(optim_result):\n",
    "#     \"\"\"Status callback durring bayesian hyperparameter search\"\"\"\n",
    "    \n",
    "#     # Get all the models tested so far in DataFrame format\n",
    "#     all_models = pd.DataFrame(bayes_cv_tuner.cv_results_)    \n",
    "    \n",
    "#     # Get current parameters and the best parameters    \n",
    "#     best_params = pd.Series(bayes_cv_tuner.best_params_)\n",
    "#     print('Model #{}\\nBest ROC-AUC: {}\\nBest params: {}\\n'.format(\n",
    "#         len(all_models),\n",
    "#         np.round(bayes_cv_tuner.best_score_, 4),\n",
    "#         bayes_cv_tuner.best_params_\n",
    "#     ))\n",
    "    \n",
    "#     # Save all model results\n",
    "#     clf_name = bayes_cv_tuner.estimator.__class__.__name__\n",
    "#     all_models.to_csv(clf_name+\"_cv_results.csv\")\n",
    "\n",
    "# result = bayes_cv_tuner.fit(X, y, callback=status_print)\n",
    "\n",
    "\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}