LG_errPredict_FinalCode_210203.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 데이콘_시스템 품질 변화로 인한 사용자 불편 예지 AI 경진대회"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 팀명: 끙정의 아이들"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Public Score: 0.83775 / 53th(12.6%)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Private Score: 0.83557 / 42th(10.0%)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary: Catb, LGBM, GBC 를 활용한 10-fold OOF 앙상블"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Library & Data Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import auc, roc_auc_score, roc_curve, precision_recall_curve, recall_score, precision_score\n",
    "\n",
    "import lightgbm\n",
    "from lightgbm import LGBMClassifier\n",
    "import catboost\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.19.3\n",
      "1.0.5\n",
      "0.23.2\n",
      "3.0.0\n",
      "0.24.1\n"
     ]
    }
   ],
   "source": [
    "print(np.__version__)\n",
    "print(pd.__version__)\n",
    "print(sklearn.__version__)\n",
    "print(lightgbm.__version__)\n",
    "print(catboost.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_P_PATH = r'C:\\Users\\Wyatt\\wyatt37/Data/systemError/train_problem_data.csv'\n",
    "TRAIN_Q_PATH = r'C:\\Users\\Wyatt\\wyatt37/Data/systemError/train_quality_data.csv'\n",
    "TRAIN_E_PATH = r'C:\\Users\\Wyatt\\wyatt37/Data/systemError/train_err_data.csv'\n",
    "TEST_Q_PATH = r'C:\\Users\\Wyatt\\wyatt37/Data/systemError/test_quality_data.csv'\n",
    "TEST_E_PATH = r'C:\\Users\\Wyatt\\wyatt37/Data/systemError/test_err_data.csv'\n",
    "SUBMISSION_PATH = r'C:\\Users\\Wyatt\\wyatt37/Data/systemError/sample_submission.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 14.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_p = pd.read_csv(TRAIN_P_PATH)\n",
    "train_q = pd.read_csv(TRAIN_Q_PATH)\n",
    "train_e = pd.read_csv(TRAIN_E_PATH)\n",
    "test_q = pd.read_csv(TEST_Q_PATH)\n",
    "test_e = pd.read_csv(TEST_E_PATH)\n",
    "submission = pd.read_csv(SUBMISSION_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((5429, 2),\n",
       " (828624, 16),\n",
       " (16554663, 6),\n",
       " (747972, 16),\n",
       " (16532648, 6),\n",
       " (14999, 2))"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_p.shape, train_q.shape, train_e.shape, test_q.shape, test_e.shape, submission.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing_problem(df, object_='binary'):\n",
    "    \"\"\"\n",
    "    definition:\n",
    "    train_problem 테이블을 받아서 target 값으로 변환\n",
    "    1. {0, 1}의 binary로 변환\n",
    "    2. {0 ~ n}의 multiclass로 변환\n",
    "    \"\"\"\n",
    "    \n",
    "    # 10001부터 24999까지의 index를 만들어줍니다.\n",
    "    user_id_idx = np.array(range(10000, 25000, 1))\n",
    "    \n",
    "    # train_new_p라는 새로운 df를 만들고 index는 위에서 만든 user_id_idx 로 지정해줍니다.\n",
    "    new_p = pd.DataFrame(index = user_id_idx)\n",
    "    new_p['target'] = 0\n",
    "    \n",
    "    if object_ == 'binary':\n",
    "        new_p.iloc[df.user_id.unique()-10000] = 1\n",
    "        new_p = new_p.reset_index()\n",
    "        new_p.rename({'index':'user_id'}, axis=1, inplace=True)\n",
    "        \n",
    "    elif object_ == 'multi':\n",
    "        # multi는 count()로 집계를 해줍니다.\n",
    "        new_p['target'] = df.groupby('user_id')['time'].count()\n",
    "        new_p = new_p.fillna(0)\n",
    "        new_p = new_p.reset_index()\n",
    "        new_p.rename({'index':'user_id'}, axis=1, inplace=True)\n",
    "        \n",
    "    return new_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_b_p = preprocessing_problem(train_p, 'binary')\n",
    "train_m_p = preprocessing_problem(train_p, 'multi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing_quality(df):\n",
    "    \"\"\"\n",
    "    definition:\n",
    "    EDA를 통해 알아낸 정보로 train_q, test_q를 정리해서 내뿜어줍니다.\n",
    "    1. qaulity_3, quality_4 를 drop 합니다.(단일 value)\n",
    "    2. qaulity_k 변수들을 정수로 encoding 합니다.\n",
    "    \"\"\"\n",
    "    # 먼저 3, 4번을 drop 합니다.\n",
    "    df.drop(['quality_3', 'quality_4'], axis=1, inplace=True)\n",
    "    \n",
    "    # qual 변수만 할당해주고, 정수로 형변환 해줍니다.\n",
    "    columns = train_q.columns[train_q.columns.str.contains('quality')]\n",
    "    # for문을 통해 각 column을 반복 작업해줍니다.\n",
    "    for col in columns:\n",
    "        if df[col].dtype == 'float64': # 기존에 float은 패스\n",
    "            df[col] = df[col].fillna(-2)\n",
    "        elif df[col].dtype == 'int64': # 기존에 int도 패스\n",
    "            df[col] = df[col].fillna(-2)\n",
    "            #print(col)\n",
    "        else:\n",
    "            # nan값이 있으면 float으로 갈 수 없으니 '-2' 으로 채워줍니다.\n",
    "            df[col] = df[col].fillna('-2')\n",
    "            df[col] = df[col].apply(lambda x: str(x).replace(',' , ''))\n",
    "            df[col] = df[col].astype(np.float64)\n",
    "        \n",
    "    # fwver 에서 null 값이 꽤 있습니다. missing으로 채우겠습니다.\n",
    "    df.fwver = df.fwver.fillna('missing')\n",
    "        \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_q = preprocessing_quality(train_q)\n",
    "test_q = preprocessing_quality(test_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing_fwver(df):\n",
    "    \"\"\"\n",
    "    definition:\n",
    "    별건 아니고, e-set에 fwver 변수에서 '10' 이라는 값이 있는데, 이게 errtype이랑 겹쳐요.\n",
    "    그래서 10을 -> 8.5.2 으로 바꿔줄 겁니다.\n",
    "    굳이 이렇게 바꾸는 이유는, 해당 fw가 8.5.3버전과 같은 model_nm을 공유하기 때문입니다.\n",
    "    \"\"\"\n",
    "    df.fwver = df.fwver.replace('10', '8.5.2')\n",
    "        \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_e = preprocessing_fwver(train_e)\n",
    "test_e = preprocessing_fwver(test_e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_datetime(df):\n",
    "    \"\"\"\n",
    "    definition:\n",
    "    'time' column이 str로 되어 있으니, datetime으로 바꿔주는 함수입니다.\n",
    "    다만 'time'양식이 pandas함수에 적용이 안되니, 강제로 슬라이싱해서 만들어줘야 합니다.\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    df['year'] = df['time'].apply(lambda x: str(x)[:4])\n",
    "    df['month'] = df['time'].apply(lambda x: str(x)[4:6])\n",
    "    df['day'] = df['time'].apply(lambda x: str(x)[6:8])\n",
    "    df['hour'] = df['time'].apply(lambda x: str(x)[8:10])\n",
    "    df['minute'] = '00' # minute을 넣어주지 않으면 datetime이 완성이 안되니, 00으로 넣어줍니다.\n",
    "    \n",
    "    df['time'] = pd.to_datetime(df.year + df.month + df.day + df.hour + df.minute)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 57.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_p = make_datetime(train_p)\n",
    "train_q = make_datetime(train_q)\n",
    "test_q = make_datetime(test_q)\n",
    "train_e = make_datetime(train_e)\n",
    "test_e = make_datetime(test_e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## from Error_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 사용한 model의 개수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_model_count = train_e[['user_id', 'model_nm']].drop_duplicates().groupby('user_id').count()\n",
    "test_model_count = test_e[['user_id', 'model_nm']].drop_duplicates().groupby('user_id').count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 사용한 fwver 의 개수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fwver_count = train_e[['user_id', 'fwver']].drop_duplicates().groupby('user_id').count()\n",
    "test_fwver_count = test_e[['user_id', 'fwver']].drop_duplicates().groupby('user_id').count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 경험한 error 의 총 개수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_err_count = train_e.groupby('user_id')['errcode'].count()\n",
    "test_err_count = test_e.groupby('user_id')['errcode'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 경험한 각 errtype의 value별 개수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 23호\n",
    "train_errcode_23 = train_e[train_e.errtype == 23][['user_id', 'errcode', 'hour']].groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "test_errcode_23 = test_e[test_e.errtype == 23][['user_id', 'errcode', 'hour']].groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "\n",
    "train_errcode_23.columns = ['UNKNOWN', 'ACTIVE', 'connLMP', 'connESTA', 'connTO', 'connLOCAL', 'STANDBY', 'TERMINATE']\n",
    "test_errcode_23.columns = ['UNKNOWN', 'ACTIVE', 'connLMP', 'connESTA', 'connTO', 'connLOCAL', 'STANDBY', 'TERMINATE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 31호\n",
    "train_errcode_31 = train_e[train_e.errtype == 31][['user_id', 'errcode', 'time']].\\\n",
    "                            groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "test_errcode_31 = test_e[test_e.errtype == 31][['user_id', 'errcode', 'time']].\\\n",
    "                            groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "train_errcode_31.columns = ['err_31_0', 'err_31_1']\n",
    "test_errcode_31.columns =['err_31_0', 'err_31_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 33호\n",
    "train_errcode_33 = train_e[train_e.errtype == 33][['user_id', 'errcode', 'hour']].groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "test_errcode_33 = test_e[test_e.errtype == 33][['user_id', 'errcode', 'hour']].groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "\n",
    "train_errcode_33.columns = ['err_33_1', 'err_33_2', 'err_33_3']\n",
    "test_errcode_33.columns = ['err_33_1', 'err_33_2', 'err_33_3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 34호\n",
    "train_errcode_34 = train_e[train_e.errtype == 34][['user_id', 'errcode', 'hour']].groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "test_errcode_34 = test_e[test_e.errtype == 34][['user_id', 'errcode', 'hour']].groupby(['user_id', 'errcode']).count().unstack().fillna(0)\n",
    "\n",
    "train_errcode_34.columns = ['err_34_1', 'err_34_2', 'err_34_3', 'err_34_4', 'err_34_5', 'err_34_6']\n",
    "test_errcode_34.columns = ['err_34_1', 'err_34_2', 'err_34_3', 'err_34_4', 'err_34_5', 'err_34_6']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### day max를 mean으로 나눈 것"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 19s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_e['day_2'] = train_e['time'].apply(lambda x: str(x)[:10])\n",
    "\n",
    "train_meanDay = (train_e\n",
    "                 .groupby(['user_id','day_2'])['day_2']\n",
    "                 .count()\n",
    "                 .unstack()\n",
    "                 .fillna(0)\n",
    "                 .loc[:, '2020-11-01':'2020-11-30']\n",
    "                 .mean(axis=1))\n",
    "\n",
    "train_maxDay = (train_e\n",
    "                .groupby(['user_id','day_2'])['day_2']\n",
    "                .count()\n",
    "                .unstack()\n",
    "                .fillna(0)\n",
    "                .loc[:, '2020-11-01':'2020-11-30']\n",
    "                .max(axis=1))\n",
    "\n",
    "train_maxBymean = pd.Series(data = np.array(train_maxDay) / np.array(train_meanDay),\n",
    "                            index = train_e.user_id.unique(),\n",
    "                            name = 'mbm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test_e['day_2'] = test_e['time'].apply(lambda x: str(x)[:10])\n",
    "\n",
    "test_meanDay = (test_e\n",
    "                .groupby(['user_id','day_2'])['day_2']\n",
    "                .count()\n",
    "                .unstack()\n",
    "                .fillna(0)\n",
    "                .loc[:, '2020-11-01':'2020-11-30']\n",
    "                .mean(axis=1))\n",
    "\n",
    "test_maxDay = (test_e\n",
    "               .groupby(['user_id','day_2'])['day_2']\n",
    "               .count()\n",
    "               .unstack()\n",
    "               .fillna(0)\n",
    "               .loc[:, '2020-11-01':'2020-11-30']\n",
    "               .max(axis=1))\n",
    "\n",
    "test_maxBymean = pd.Series(data = np.array(test_maxDay) / np.array(test_meanDay),\n",
    "                           index = test_e.user_id.unique(),\n",
    "                           name = 'mbm')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## from Quality_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 각 quality의 표준편차"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_qual_std = train_q.groupby(['user_id']).std()\n",
    "test_qual_std = test_q.groupby(['user_id']).std()\n",
    "\n",
    "new_columns = ['q_std_0', 'q_std_1', 'q_std_2', 'q_std_5', 'q_std_6', 'q_std_7', 'q_std_8', 'q_std_9', 'q_std_10','q_std_11', 'q_std_12']\n",
    "\n",
    "train_qual_std.columns = new_columns\n",
    "test_qual_std.columns = new_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 기록한 quality log의 개수(12개당 1번)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_qual_log = train_q.groupby('user_id')['time'].count()/12\n",
    "test_qual_log = test_q.groupby('user_id')['time'].count()/12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### quality당 순수 개수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 값을 만들기 위해 새로운 df를 받아옵니다.\n",
    "train_q_temp = train_q.copy()\n",
    "test_q_temp = test_q.copy()\n",
    "\n",
    "# 0 값을 전부 nan 값으로 바꿔줍니다.\n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    train_q_temp.loc[train_q_temp['quality_{}'.format(i)] == 0, 'quality_{}'.format(i)] = np.nan\n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    test_q_temp.loc[test_q_temp['quality_{}'.format(i)] == 0, 'quality_{}'.format(i)] = np.nan\n",
    "\n",
    "# 필요없는 변수들을 버려줍니다.\n",
    "train_q_temp.drop(['time', 'fwver', 'year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=True)\n",
    "test_q_temp.drop(['time', 'fwver', 'year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=True)\n",
    "\n",
    "# 그룹바이 카운트 해줍니다.\n",
    "train_qual_counts = train_q_temp.groupby('user_id').count()\n",
    "test_qual_counts = test_q_temp.groupby('user_id').count()\n",
    "\n",
    "# 겹치는 컬럼명을 바꿔줍니다.\n",
    "train_qual_counts.columns = ['q_c_0', 'q_c_1', 'q_c_2', 'q_c_5', 'q_c_6',\n",
    "                             'q_c_7', 'q_c_8', 'q_c_9', 'q_c_10', 'q_c_11', 'q_c_12']\n",
    "test_qual_counts.columns = ['q_c_0', 'q_c_1', 'q_c_2', 'q_c_5', 'q_c_6',\n",
    "                             'q_c_7', 'q_c_8', 'q_c_9', 'q_c_10', 'q_c_11', 'q_c_12']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### quality당 음수, 0에 대한 count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_q_temp = train_q.copy()\n",
    "test_q_temp = test_q.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_q_temp.drop(['time', 'fwver', 'year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=True)\n",
    "test_q_temp.drop(['time', 'fwver', 'year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0 값만 count를 위해서 음수와 양수를 전부 nan으로 만들겠습니다.\n",
    "\n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    train_q_temp.loc[train_q_temp['quality_{}'.format(i)] < 0, 'quality_{}'.format(i)] = np.nan\n",
    "    \n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    test_q_temp.loc[test_q_temp['quality_{}'.format(i)] < 0, 'quality_{}'.format(i)] = np.nan\n",
    "    \n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    train_q_temp.loc[train_q_temp['quality_{}'.format(i)] > 0, 'quality_{}'.format(i)] = np.nan\n",
    "    \n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    test_q_temp.loc[test_q_temp['quality_{}'.format(i)] > 0, 'quality_{}'.format(i)] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_qual_zeroCount = train_q_temp.groupby('user_id').count()\n",
    "test_qual_zeroCount = test_q_temp.groupby('user_id').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_columns = ['q_z_c_0', 'q_z_c_1', 'q_z_c_2', 'q_z_c_5', 'q_z_c_6', 'q_z_c_7', 'q_z_c_8', 'q_z_c_9', 'q_z_c_10','q_z_c_11', 'q_z_c_12']\n",
    "\n",
    "train_qual_zeroCount.columns = new_columns\n",
    "test_qual_zeroCount.columns = new_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_q_temp = train_q.copy()\n",
    "test_q_temp = test_q.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_q_temp.drop(['time', 'fwver', 'year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=True)\n",
    "test_q_temp.drop(['time', 'fwver', 'year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 음수 값만 count를 위해서 음수와 0을 전부 nan으로 만들겠습니다.\n",
    "\n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    train_q_temp.loc[train_q_temp['quality_{}'.format(i)] >= 0, 'quality_{}'.format(i)] = np.nan\n",
    "    \n",
    "for i in [0, 1, 2, 5, 6, 7, 8, 9, 10, 11, 12]:\n",
    "    test_q_temp.loc[test_q_temp['quality_{}'.format(i)] >= 0, 'quality_{}'.format(i)] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_qual_negaCount = train_q_temp.groupby('user_id').count()\n",
    "test_qual_negaCount = test_q_temp.groupby('user_id').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_columns = ['q_n_c_0', 'q_n_c_1', 'q_n_c_2', 'q_n_c_5', 'q_n_c_6', 'q_n_c_7', 'q_n_c_8', 'q_n_c_9', 'q_n_c_10','q_n_c_11', 'q_n_c_12']\n",
    "\n",
    "train_qual_negaCount.columns = new_columns\n",
    "test_qual_negaCount.columns = new_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### quality를 sum으로 groupby"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_each_quality_sum = train_q.groupby('user_id').sum().loc[:, 'quality_0':'quality_12']\n",
    "test_each_quality_sum = test_q.groupby('user_id').sum().loc[:, 'quality_0':'quality_12']\n",
    "\n",
    "quality_sum_colnms = ['quality_0_sum', 'quality_1_sum', 'quality_2_sum', 'quality_5_sum', 'quality_6_sum', \n",
    "                      'quality_7_sum', 'quality_8_sum', 'quality_9_sum','quality_10_sum', 'quality_11_sum', \n",
    "                      'quality_12_sum']\n",
    "\n",
    "train_each_quality_sum.columns = quality_sum_colnms\n",
    "test_each_quality_sum.columns = quality_sum_colnms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## from both"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### time에 대한 유저별 표준편차"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_to_seconds(x):\n",
    "    return time.mktime(x.timetuple())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_e['time_sec'] = train_e.time.apply(lambda x: time_to_seconds(x))\n",
    "test_e['time_sec'] = test_e.time.apply(lambda x: time_to_seconds(x))\n",
    "train_q['time_sec'] = train_q.time.apply(lambda x: time_to_seconds(x))\n",
    "test_q['time_sec'] = test_q.time.apply(lambda x: time_to_seconds(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 480 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_err_timestd = train_e.groupby(['user_id'])['time_sec'].std()\n",
    "test_err_timestd = test_e.groupby(['user_id'])['time_sec'].std()\n",
    "train_err_timestd = train_err_timestd.rename(level = 0, index = 't_e_std') \n",
    "test_err_timestd = test_err_timestd.rename(level = 0, index = 't_e_std') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 119 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_qual_timestd = (train_q[['user_id', 'time_sec']].drop_duplicates()).groupby(['user_id']).std()\n",
    "test_qual_timestd = (test_q[['user_id', 'time_sec']].drop_duplicates()).groupby(['user_id']).std()\n",
    "train_qual_timestd.columns = ['t_q_std']\n",
    "test_qual_timestd.columns = ['t_q_std']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# errtype을 유저별로 카운트 해줍니다.\n",
    "\n",
    "X = train_e.groupby(['user_id', 'errtype'])['errcode'].count().unstack().fillna(0)\n",
    "y = test_e.groupby(['user_id', 'errtype'])['errcode'].count().unstack().fillna(0)\n",
    "\n",
    "X.columns = ['et_1', 'et_2', 'et_3', 'et_4', 'et_5', 'et_6', 'et_7', 'et_8', 'et_9', 'et_10', 'et_11', 'et_12', 'et_13',\n",
    " 'et_14', 'et_15', 'et_16', 'et_17', 'et_18', 'et_19', 'et_20', 'et_21', 'et_22', 'et_23', 'et_24', 'et_25',\n",
    " 'et_26', 'et_27', 'et_28', 'et_30', 'et_31', 'et_32', 'et_33', 'et_34', 'et_35', 'et_36', 'et_37', 'et_38',\n",
    " 'et_39', 'et_40', 'et_41', 'et_42']\n",
    "y.columns = ['et_1', 'et_2', 'et_3', 'et_4', 'et_5', 'et_6', 'et_7', 'et_8', 'et_9', 'et_10', 'et_11', 'et_12', 'et_13',\n",
    " 'et_14', 'et_15', 'et_16', 'et_17', 'et_18', 'et_19', 'et_20', 'et_21', 'et_22', 'et_23', 'et_24', 'et_25',\n",
    " 'et_26', 'et_27', 'et_28', 'et_30', 'et_31', 'et_32', 'et_33', 'et_34', 'et_35', 'et_36', 'et_37', 'et_38',\n",
    " 'et_39', 'et_40', 'et_41', 'et_42']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 하나 사라진 유저를 채워주는 코드입니다.\n",
    "y = y.reindex(pd.RangeIndex(y.index.max() + 1)).ffill(0)[30000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((15000, 41), (14999, 41))"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15000, 122)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = pd.concat([X,\n",
    "               train_err_count, # 유저가 기록한 총 err수\n",
    "               train_fwver_count, # 유저가 사용한 fw수\n",
    "               train_model_count, # 유저가 사용한 model 수\n",
    "               train_qual_std, # 각 퀄리티에 대한 유저별 편차\n",
    "               train_qual_log, # 유저별 퀄리티 로그의 수\n",
    "               train_errcode_23, # 23호 에러타입의 밸류별 개수\n",
    "               train_errcode_33, # 33호 상동\n",
    "               train_errcode_34, # 34호 상동\n",
    "               train_qual_counts, # 각 퀄리티에서 0을 제외한 카운트\n",
    "               train_qual_negaCount, # 각 퀄리티에 대해 음수만 카운트\n",
    "               train_qual_zeroCount, # 각 퀄리티에 대해 0.만 카운트\n",
    "               train_err_timestd,\n",
    "               train_qual_timestd,\n",
    "               train_each_quality_sum,\n",
    "               train_errcode_31, # 31호 상동\n",
    "               train_maxBymean # time\n",
    "              ], axis=1).fillna(0)\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14999, 122)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = pd.concat([y,\n",
    "               test_err_count, # 유저가 기록한 총 err수\n",
    "               test_fwver_count, # 유저가 사용한 fw수\n",
    "               test_model_count, # 유저가 사용한 model 수\n",
    "               test_qual_std, # 각 퀄리티에 대한 유저별 편차\n",
    "               test_qual_log, # 유저별 퀄리티 로그의 수\n",
    "               test_errcode_23, # 23호 에러타입의 밸류별 개수\n",
    "               test_errcode_33, # 33호 상동\n",
    "               test_errcode_34, # 34호 상동\n",
    "               test_qual_counts, # 각 퀄리티에서 0을 제외한 카운트\n",
    "               test_qual_negaCount, # 각 퀄리티에 대해 음수만 카운트\n",
    "               test_qual_zeroCount, # 각 퀄리티에 대해 0.만 카운트\n",
    "               test_err_timestd,\n",
    "               test_qual_timestd,\n",
    "               test_each_quality_sum,\n",
    "               test_errcode_31, # 31호 상동\n",
    "               test_maxBymean, # time\n",
    "              ], axis=1).fillna(0)\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((15000, 120), (14999, 120))"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 일부 다중공선성을 띄는 변수들을 제거해줍니다.\n",
    "X.drop(['et_20', 'et_36'], axis=1, inplace = True)\n",
    "y.drop(['et_20', 'et_36'], axis=1, inplace = True)\n",
    "\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuned Catb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def catb_fold_train_pred(train_x, train_y):\n",
    "\n",
    "    # Train\n",
    "    models     = []\n",
    "    recalls    = []\n",
    "    precisions = []\n",
    "    auc_scores   = []\n",
    "    threshold = 0.5\n",
    "    # 파라미터 설정\n",
    "    params =      {\n",
    "                    'nan_mode': 'Min',\n",
    "                    'eval_metric': 'Logloss',\n",
    "                    'iterations': 1000,\n",
    "                    'sampling_frequency': 'PerTree',\n",
    "                    'leaf_estimation_method': 'Newton',\n",
    "                    'grow_policy': 'SymmetricTree',\n",
    "                    'penalties_coefficient': 1,\n",
    "                    'boosting_type': 'Plain',\n",
    "                    'model_shrink_mode': 'Constant',\n",
    "                    'feature_border_type': 'GreedyLogSum',\n",
    "                    'l2_leaf_reg': 3,\n",
    "                    'random_strength': 1,\n",
    "                    'rsm': 1,\n",
    "                    'boost_from_average': False,\n",
    "                    'model_size_reg': 0.5,\n",
    "                    'subsample': 0.800000011920929,\n",
    "                    'use_best_model': False,\n",
    "                    'class_names': [0, 1],\n",
    "                    'random_seed': 2584,\n",
    "                    'depth': 6,\n",
    "                    'posterior_sampling': False,\n",
    "                    'border_count': 254,\n",
    "                    'classes_count': 0,\n",
    "                    'auto_class_weights': 'None',\n",
    "                    'sparse_features_conflict_fraction': 0,\n",
    "                    'leaf_estimation_backtracking': 'AnyImprovement',\n",
    "                    'best_model_min_trees': 1,\n",
    "                    'model_shrink_rate': 0,\n",
    "                    'min_data_in_leaf': 1,\n",
    "                    'loss_function': 'Logloss',\n",
    "                    'learning_rate': 0.028116999194025993,\n",
    "                    'score_function': 'Cosine',\n",
    "                    'task_type': 'CPU',\n",
    "                    'leaf_estimation_iterations': 10,\n",
    "                    'bootstrap_type': 'MVS',\n",
    "                    'max_leaves': 64\n",
    "                    }\n",
    "    #-------------------------------------------------------------------------------------\n",
    "    # 5 Kfold cross validation\n",
    "    s_fold = StratifiedKFold(n_splits=5, shuffle=True ,random_state=42)    \n",
    "\n",
    "    for train_idx, val_idx in s_fold.split(train_x, train_y):\n",
    "\n",
    "        # split train, validation set\n",
    "        X = train_x.iloc[train_idx]\n",
    "        y = train_y.iloc[train_idx]\n",
    "        valid_x = train_x.iloc[val_idx]\n",
    "        valid_y = train_y.iloc[val_idx]\n",
    "\n",
    "        #run traning\n",
    "        model = CatBoostClassifier(**params, verbose=0)\n",
    "        model.fit(X, y)\n",
    "\n",
    "        # cal valid prediction\n",
    "        valid_prob = model.predict(valid_x)\n",
    "        valid_pred = np.where(valid_prob > threshold, 1, 0)\n",
    "\n",
    "        # cal scores\n",
    "        recall    = recall_score(    valid_y, valid_pred)\n",
    "        precision = precision_score( valid_y, valid_pred)\n",
    "        auc_score = roc_auc_score(   valid_y, valid_prob)\n",
    "\n",
    "        # append scores\n",
    "        models.append(model)\n",
    "        recalls.append(recall)\n",
    "        precisions.append(precision)\n",
    "        auc_scores.append(auc_score)\n",
    "        \n",
    "    return models, auc_scores, recalls, precisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7301499999999999\n"
     ]
    }
   ],
   "source": [
    "# loss 비교를 위해 지우지 않습니다.\n",
    "# 최고점 모델입니다.\n",
    "models, auc_scores, _, _ = catb_fold_train_pred(X, train_b_p.target)\n",
    "print(np.mean(auc_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_y_list = []\n",
    "for model in models:\n",
    "    pred_y = model.predict_proba(y)\n",
    "    pred_y_list.append(pred_y[:, 1].reshape(-1,1))\n",
    "    \n",
    "pred_ensemble_catb = np.mean(pred_y_list, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.93218016],\n",
       "       [0.22254721],\n",
       "       [0.48239791],\n",
       "       ...,\n",
       "       [0.70990508],\n",
       "       [0.8953148 ],\n",
       "       [0.43733988]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_catb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuned LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgbm_fold_train_pred(train_x, train_y):\n",
    "\n",
    "    # Train\n",
    "    models     = []\n",
    "    recalls    = []\n",
    "    precisions = []\n",
    "    auc_scores   = []\n",
    "    threshold = 0.5\n",
    "    # 파라미터 설정\n",
    "    params =      {\n",
    "                    'boosting_type':'gbdt', \n",
    "                    'class_weight':None,\n",
    "                    'colsample_bytree':1.0,\n",
    "                    'importance_type':'split',\n",
    "                    'learning_rate':0.1,\n",
    "                    'max_depth':-1,\n",
    "                    'min_child_samples':20,\n",
    "                    'min_child_weight':0.001,\n",
    "                    'min_split_gain':0.0,\n",
    "                    'n_estimators':100,\n",
    "                    'n_jobs':-1,\n",
    "                    'num_leaves':31,\n",
    "                    'objective':None,\n",
    "                    'random_state':2584,\n",
    "                    'reg_alpha':0.0,\n",
    "                    'reg_lambda':0.0,\n",
    "                    'silent':True,\n",
    "                    'subsample':1.0,\n",
    "                    'subsample_for_bin':200000,\n",
    "                    'subsample_freq':0\n",
    "                    }\n",
    "    #-------------------------------------------------------------------------------------\n",
    "    # 5 Kfold cross validation\n",
    "    s_fold = StratifiedKFold(n_splits=5, shuffle=True ,random_state=42)    \n",
    "\n",
    "    for train_idx, val_idx in s_fold.split(train_x, train_y):\n",
    "\n",
    "        # split train, validation set\n",
    "        X = train_x.iloc[train_idx]\n",
    "        y = train_y.iloc[train_idx]\n",
    "        valid_x = train_x.iloc[val_idx]\n",
    "        valid_y = train_y.iloc[val_idx]\n",
    "\n",
    "        #run traning\n",
    "        model = LGBMClassifier(**params)\n",
    "        model.fit(X, y)\n",
    "\n",
    "        # cal valid prediction\n",
    "        valid_prob = model.predict(valid_x)\n",
    "        valid_pred = np.where(valid_prob > threshold, 1, 0)\n",
    "\n",
    "        # cal scores\n",
    "        recall    = recall_score(    valid_y, valid_pred)\n",
    "        precision = precision_score( valid_y, valid_pred)\n",
    "        auc_score = roc_auc_score(   valid_y, valid_prob)\n",
    "\n",
    "        # append scores\n",
    "        models.append(model)\n",
    "        recalls.append(recall)\n",
    "        precisions.append(precision)\n",
    "        auc_scores.append(auc_score)\n",
    "        \n",
    "    return models, auc_scores, recalls, precisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.73295\n"
     ]
    }
   ],
   "source": [
    "models, auc_scores, _, _ = lgbm_fold_train_pred(X, train_b_p.target)\n",
    "print(np.mean(auc_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_y_list = []\n",
    "for model in models:\n",
    "    pred_y = model.predict_proba(y)\n",
    "    pred_y_list.append(pred_y[:, 1].reshape(-1,1))\n",
    "    \n",
    "pred_ensemble_lgbm = np.mean(pred_y_list, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.94799909],\n",
       "       [0.22247434],\n",
       "       [0.48895694],\n",
       "       ...,\n",
       "       [0.74520416],\n",
       "       [0.87992041],\n",
       "       [0.34860365]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_lgbm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuned GBC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbc = GradientBoostingClassifier(\n",
    "                                ccp_alpha=0.0,\n",
    "                                criterion='friedman_mse',\n",
    "                                init=None,\n",
    "                                learning_rate=0.1,\n",
    "                                loss='deviance',\n",
    "                                max_depth=3,\n",
    "                                max_features=None,\n",
    "                                max_leaf_nodes=None,\n",
    "                                min_impurity_decrease=0.0,\n",
    "                                min_impurity_split=None,\n",
    "                                min_samples_leaf=1,\n",
    "                                min_samples_split=2,\n",
    "                                min_weight_fraction_leaf=0.0,\n",
    "                                n_estimators=100,\n",
    "                                n_iter_no_change=None,\n",
    "                                presort='deprecated',\n",
    "                                random_state=2584,\n",
    "                                subsample=1.0,\n",
    "                                tol=0.0001,\n",
    "                                validation_fraction=0.1,\n",
    "                                verbose=0,\n",
    "                                warm_start=False\n",
    "                            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gbc_fold_train_pred(train_x, train_y, N_SPLIT=5):\n",
    "\n",
    "    # Train\n",
    "    models     = []\n",
    "    recalls    = []\n",
    "    precisions = []\n",
    "    auc_scores   = []\n",
    "    threshold = 0.5\n",
    "\n",
    "    k_fold = StratifiedKFold(n_splits=N_SPLIT, shuffle=True, random_state=42)\n",
    "    for train_idx, val_idx in k_fold.split(train_x, train_y):\n",
    "\n",
    "        # split train, validation set\n",
    "        X = train_x.iloc[train_idx]\n",
    "        y = train_y.iloc[train_idx]\n",
    "        valid_x = train_x.iloc[val_idx]\n",
    "        valid_y = train_y.iloc[val_idx]\n",
    "\n",
    "        #run traning\n",
    "        model = gbc.fit(X, y)\n",
    "\n",
    "        # cal valid prediction\n",
    "        valid_prob = model.predict(valid_x)\n",
    "        valid_pred = np.where(valid_prob > threshold, 1, 0)\n",
    "\n",
    "        # cal scores\n",
    "        recall    = recall_score(    valid_y, valid_pred)\n",
    "        precision = precision_score( valid_y, valid_pred)\n",
    "        auc_score = roc_auc_score(   valid_y, valid_prob)\n",
    "        \n",
    "        # append scores\n",
    "        models.append(model)\n",
    "        recalls.append(recall)\n",
    "        precisions.append(precision)\n",
    "        auc_scores.append(auc_score)\n",
    "    \n",
    "    return models, auc_scores, recalls, precisions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.72455\n",
      "Wall time: 36.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "models, auc_scores, _, _ = gbc_fold_train_pred(X, train_b_p.target)\n",
    "print(np.mean(auc_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_y_list = []\n",
    "for model in models:\n",
    "    pred_y = model.predict_proba(y)\n",
    "    pred_y_list.append(pred_y[:, 1].reshape(-1,1))\n",
    "    \n",
    "pred_ensemble_gbc = np.mean(pred_y_list, axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.94729912],\n",
       "       [0.20899576],\n",
       "       [0.48470505],\n",
       "       ...,\n",
       "       [0.75559137],\n",
       "       [0.87481054],\n",
       "       [0.34822194]])"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_gbc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.93218016],\n",
       "       [0.22254721],\n",
       "       [0.48239791],\n",
       "       ...,\n",
       "       [0.70990508],\n",
       "       [0.8953148 ],\n",
       "       [0.43733988]])"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_catb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.94799909],\n",
       "       [0.22247434],\n",
       "       [0.48895694],\n",
       "       ...,\n",
       "       [0.74520416],\n",
       "       [0.87992041],\n",
       "       [0.34860365]])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_lgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.94729912],\n",
       "       [0.20899576],\n",
       "       [0.48470505],\n",
       "       ...,\n",
       "       [0.75559137],\n",
       "       [0.87481054],\n",
       "       [0.34822194]])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_gbc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>problem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30000</td>\n",
       "      <td>0.942493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30001</td>\n",
       "      <td>0.218006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30002</td>\n",
       "      <td>0.485353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30003</td>\n",
       "      <td>0.791206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30004</td>\n",
       "      <td>0.898756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14994</th>\n",
       "      <td>44994</td>\n",
       "      <td>0.206501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14995</th>\n",
       "      <td>44995</td>\n",
       "      <td>0.288811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14996</th>\n",
       "      <td>44996</td>\n",
       "      <td>0.736900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14997</th>\n",
       "      <td>44997</td>\n",
       "      <td>0.883349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14998</th>\n",
       "      <td>44998</td>\n",
       "      <td>0.378055</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14999 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id   problem\n",
       "0        30000  0.942493\n",
       "1        30001  0.218006\n",
       "2        30002  0.485353\n",
       "3        30003  0.791206\n",
       "4        30004  0.898756\n",
       "...        ...       ...\n",
       "14994    44994  0.206501\n",
       "14995    44995  0.288811\n",
       "14996    44996  0.736900\n",
       "14997    44997  0.883349\n",
       "14998    44998  0.378055\n",
       "\n",
       "[14999 rows x 2 columns]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_ensemble_best = (\n",
    "                            pred_ensemble_catb +\n",
    "                            pred_ensemble_lgbm  +\n",
    "                            pred_ensemble_gbc\n",
    "                        ) / 3\n",
    "\n",
    "submission.problem = pred_ensemble_best\n",
    "submission"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "끝"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}