Submit1_Stacking ensemble_Full_Version.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h6h89BPK1CaA",
    "outputId": "a8843b6a-2099-4d0f-a975-dfaec0a0dd94"
   },
   "source": [
    "# Stacking Ensenble을 활용한 자율주행 센서 안테나 성능예측\n",
    "> 팀명 : 될때까지간다리\n",
    ">\n",
    "> 작성일 : '22.08.31\n",
    ">\n",
    "> 개발환경 : Jupyter Notebook\n",
    ">\n",
    "> 결과 : 상위 4%"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Development Environment Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V1s0gwOEkAYi",
    "outputId": "929c6c3e-2a40-49e6-8944-331578af424a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: optuna in c:\\dev\\miniconda3\\lib\\site-packages (2.10.1)\n",
      "Requirement already satisfied: cliff in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (4.0.0)\n",
      "Requirement already satisfied: colorlog in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (6.6.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (21.3)\n",
      "Requirement already satisfied: PyYAML in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (6.0)\n",
      "Requirement already satisfied: alembic in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (1.8.1)\n",
      "Requirement already satisfied: cmaes>=0.8.2 in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (0.8.2)\n",
      "Requirement already satisfied: numpy in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (1.23.2)\n",
      "Requirement already satisfied: sqlalchemy>=1.1.0 in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (1.4.40)\n",
      "Requirement already satisfied: scipy!=1.4.0 in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (1.9.0)\n",
      "Requirement already satisfied: tqdm in c:\\dev\\miniconda3\\lib\\site-packages (from optuna) (4.63.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\dev\\miniconda3\\lib\\site-packages (from packaging>=20.0->optuna) (3.0.4)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in c:\\dev\\miniconda3\\lib\\site-packages (from sqlalchemy>=1.1.0->optuna) (1.1.2)\n",
      "Requirement already satisfied: Mako in c:\\dev\\miniconda3\\lib\\site-packages (from alembic->optuna) (1.2.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in c:\\dev\\miniconda3\\lib\\site-packages (from cliff->optuna) (4.12.0)\n",
      "Requirement already satisfied: PrettyTable>=0.7.2 in c:\\dev\\miniconda3\\lib\\site-packages (from cliff->optuna) (3.3.0)\n",
      "Requirement already satisfied: stevedore>=2.0.1 in c:\\dev\\miniconda3\\lib\\site-packages (from cliff->optuna) (4.0.0)\n",
      "Requirement already satisfied: autopage>=0.4.0 in c:\\dev\\miniconda3\\lib\\site-packages (from cliff->optuna) (0.5.1)\n",
      "Requirement already satisfied: cmd2>=1.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from cliff->optuna) (2.4.2)\n",
      "Requirement already satisfied: attrs>=16.3.0 in c:\\dev\\miniconda3\\lib\\site-packages (from cmd2>=1.0.0->cliff->optuna) (21.4.0)\n",
      "Requirement already satisfied: pyperclip>=1.6 in c:\\dev\\miniconda3\\lib\\site-packages (from cmd2>=1.0.0->cliff->optuna) (1.8.2)\n",
      "Requirement already satisfied: wcwidth>=0.1.7 in c:\\dev\\miniconda3\\lib\\site-packages (from cmd2>=1.0.0->cliff->optuna) (0.2.5)\n",
      "Requirement already satisfied: pyreadline3 in c:\\dev\\miniconda3\\lib\\site-packages (from cmd2>=1.0.0->cliff->optuna) (3.4.1)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\dev\\miniconda3\\lib\\site-packages (from importlib-metadata>=4.4->cliff->optuna) (3.8.1)\n",
      "Requirement already satisfied: pbr!=2.1.0,>=2.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from stevedore>=2.0.1->cliff->optuna) (5.10.0)\n",
      "Requirement already satisfied: colorama in c:\\dev\\miniconda3\\lib\\site-packages (from colorlog->optuna) (0.4.4)\n",
      "Requirement already satisfied: MarkupSafe>=0.9.2 in c:\\dev\\miniconda3\\lib\\site-packages (from Mako->alembic->optuna) (2.1.1)\n",
      "Requirement already satisfied: catboost in c:\\dev\\miniconda3\\lib\\site-packages (1.0.6)\n",
      "Requirement already satisfied: scipy in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (1.9.0)\n",
      "Requirement already satisfied: graphviz in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (0.20.1)\n",
      "Requirement already satisfied: numpy>=1.16.0 in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (1.23.2)\n",
      "Requirement already satisfied: matplotlib in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (3.5.3)\n",
      "Requirement already satisfied: pandas>=0.24.0 in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (1.4.3)\n",
      "Requirement already satisfied: six in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (1.16.0)\n",
      "Requirement already satisfied: plotly in c:\\dev\\miniconda3\\lib\\site-packages (from catboost) (5.10.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\dev\\miniconda3\\lib\\site-packages (from pandas>=0.24.0->catboost) (2022.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\dev\\miniconda3\\lib\\site-packages (from pandas>=0.24.0->catboost) (2.8.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib->catboost) (9.2.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib->catboost) (4.35.0)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib->catboost) (0.11.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib->catboost) (21.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib->catboost) (1.4.4)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib->catboost) (3.0.4)\n",
      "Requirement already satisfied: tenacity>=6.2.0 in c:\\dev\\miniconda3\\lib\\site-packages (from plotly->catboost) (8.0.1)\n",
      "Requirement already satisfied: skranger in c:\\dev\\miniconda3\\lib\\site-packages (0.8.0)\n",
      "Requirement already satisfied: scikit-learn>=1.0 in c:\\dev\\miniconda3\\lib\\site-packages (from skranger) (1.1.2)\n",
      "Requirement already satisfied: joblib>=1.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn>=1.0->skranger) (1.1.0)\n",
      "Requirement already satisfied: scipy>=1.3.2 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn>=1.0->skranger) (1.9.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn>=1.0->skranger) (3.1.0)\n",
      "Requirement already satisfied: numpy>=1.17.3 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn>=1.0->skranger) (1.23.2)\n",
      "Requirement already satisfied: ngboost in c:\\dev\\miniconda3\\lib\\site-packages (0.3.12)\n",
      "Requirement already satisfied: scipy>=1.3 in c:\\dev\\miniconda3\\lib\\site-packages (from ngboost) (1.9.0)\n",
      "Requirement already satisfied: tqdm>=4.3 in c:\\dev\\miniconda3\\lib\\site-packages (from ngboost) (4.63.0)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\dev\\miniconda3\\lib\\site-packages (from ngboost) (1.23.2)\n",
      "Requirement already satisfied: lifelines>=0.25 in c:\\dev\\miniconda3\\lib\\site-packages (from ngboost) (0.27.1)\n",
      "Requirement already satisfied: scikit-learn>=0.21 in c:\\dev\\miniconda3\\lib\\site-packages (from ngboost) (1.1.2)\n",
      "Requirement already satisfied: autograd-gamma>=0.3 in c:\\dev\\miniconda3\\lib\\site-packages (from lifelines>=0.25->ngboost) (0.5.0)\n",
      "Requirement already satisfied: pandas>=1.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from lifelines>=0.25->ngboost) (1.4.3)\n",
      "Requirement already satisfied: autograd>=1.3 in c:\\dev\\miniconda3\\lib\\site-packages (from lifelines>=0.25->ngboost) (1.4)\n",
      "Requirement already satisfied: matplotlib>=3.0 in c:\\dev\\miniconda3\\lib\\site-packages (from lifelines>=0.25->ngboost) (3.5.3)\n",
      "Requirement already satisfied: formulaic>=0.2.2 in c:\\dev\\miniconda3\\lib\\site-packages (from lifelines>=0.25->ngboost) (0.4.0)\n",
      "Requirement already satisfied: future>=0.15.2 in c:\\dev\\miniconda3\\lib\\site-packages (from autograd>=1.3->lifelines>=0.25->ngboost) (0.18.2)\n",
      "Requirement already satisfied: wrapt>=1.0 in c:\\dev\\miniconda3\\lib\\site-packages (from formulaic>=0.2.2->lifelines>=0.25->ngboost) (1.14.1)\n",
      "Requirement already satisfied: interface-meta<2.0.0,>=1.2.0 in c:\\dev\\miniconda3\\lib\\site-packages (from formulaic>=0.2.2->lifelines>=0.25->ngboost) (1.3.0)\n",
      "Requirement already satisfied: astor>=0.8 in c:\\dev\\miniconda3\\lib\\site-packages (from formulaic>=0.2.2->lifelines>=0.25->ngboost) (0.8.1)\n",
      "Requirement already satisfied: typing-extensions<5.0.0,>=4.2.0 in c:\\dev\\miniconda3\\lib\\site-packages (from formulaic>=0.2.2->lifelines>=0.25->ngboost) (4.3.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (21.3)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (3.0.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (2.8.2)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (4.35.0)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (0.11.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\dev\\miniconda3\\lib\\site-packages (from matplotlib>=3.0->lifelines>=0.25->ngboost) (9.2.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\dev\\miniconda3\\lib\\site-packages (from pandas>=1.0.0->lifelines>=0.25->ngboost) (2022.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\dev\\miniconda3\\lib\\site-packages (from python-dateutil>=2.7->matplotlib>=3.0->lifelines>=0.25->ngboost) (1.16.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn>=0.21->ngboost) (3.1.0)\n",
      "Requirement already satisfied: joblib>=1.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn>=0.21->ngboost) (1.1.0)\n",
      "Requirement already satisfied: colorama in c:\\dev\\miniconda3\\lib\\site-packages (from tqdm>=4.3->ngboost) (0.4.4)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lightgbm in c:\\dev\\miniconda3\\lib\\site-packages (3.3.2)\n",
      "Requirement already satisfied: wheel in c:\\dev\\miniconda3\\lib\\site-packages (from lightgbm) (0.37.1)\n",
      "Requirement already satisfied: scipy in c:\\dev\\miniconda3\\lib\\site-packages (from lightgbm) (1.9.0)\n",
      "Requirement already satisfied: scikit-learn!=0.22.0 in c:\\dev\\miniconda3\\lib\\site-packages (from lightgbm) (1.1.2)\n",
      "Requirement already satisfied: numpy in c:\\dev\\miniconda3\\lib\\site-packages (from lightgbm) (1.23.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn!=0.22.0->lightgbm) (3.1.0)\n",
      "Requirement already satisfied: joblib>=1.0.0 in c:\\dev\\miniconda3\\lib\\site-packages (from scikit-learn!=0.22.0->lightgbm) (1.1.0)\n",
      "Requirement already satisfied: hyperopt in c:\\dev\\miniconda3\\lib\\site-packages (0.2.7)\n",
      "Requirement already satisfied: numpy in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (1.23.2)\n",
      "Requirement already satisfied: cloudpickle in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (2.1.0)\n",
      "Requirement already satisfied: six in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (1.16.0)\n",
      "Requirement already satisfied: tqdm in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (4.63.0)\n",
      "Requirement already satisfied: py4j in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (0.10.9.7)\n",
      "Requirement already satisfied: scipy in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (1.9.0)\n",
      "Requirement already satisfied: future in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (0.18.2)\n",
      "Requirement already satisfied: networkx>=2.2 in c:\\dev\\miniconda3\\lib\\site-packages (from hyperopt) (2.8.6)\n",
      "Requirement already satisfied: colorama in c:\\dev\\miniconda3\\lib\\site-packages (from tqdm->hyperopt) (0.4.4)\n"
     ]
    }
   ],
   "source": [
    "# hyper parameter tuning을 위한 패키지 설치\n",
    "!pip install optuna\n",
    "!pip install catboost\n",
    "!pip install skranger\n",
    "!pip install ngboost\n",
    "!pip install lightgbm\n",
    "!pip install hyperopt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "pcG2aSVkkAVz"
   },
   "outputs": [],
   "source": [
    "# 기본 modules\n",
    "import pandas as pd\n",
    "import random\n",
    "import os\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import numpy as np\n",
    "import tqdm\n",
    "\n",
    "# 머신러닝 modules\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split, KFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from lightgbm import LGBMRegressor\n",
    "from ngboost import NGBRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, ExtraTreesRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.linear_model import ElasticNet, LinearRegression, Lasso, Ridge\n",
    "from catboost import CatBoostRegressor, Pool\n",
    "from skranger.ensemble import RangerForestRegressor\n",
    "from sklearn.neighbors import RadiusNeighborsRegressor\n",
    "from ngboost.scores import LogScore\n",
    "\n",
    "from hyperopt import fmin, hp, tpe\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "# 모듈화된 함수 사용\n",
    "import utils.preprocessing as preprocessing\n",
    "import utils.utils as utils\n",
    "import utils.stacking as stk\n",
    "import utils.params as params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# seed 고정\n",
    "def seed_everything(seed): \n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Config:\n",
    "    seed = 42\n",
    "    epochs = 1 #200\n",
    "    cv=2 #10\n",
    "    test_size = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Y_Feature별 NRMSE의 총합\n",
    "def lg_nrmse(gt, preds):\n",
    "    \"\"\"\n",
    "    @Description: Metric used in this project\n",
    "    @Params1: gt, pandas dataframe\n",
    "    @Param2: preds, pandas dataframe\n",
    "    @Return: nrmse score\n",
    "    \"\"\"\n",
    "    preds = pd.DataFrame(preds)\n",
    "    all_nrmse = []\n",
    "    for idx in range(0,14):\n",
    "        rmse = mean_squared_error(gt.iloc[:,idx], preds.iloc[:,idx], squared=False)\n",
    "        nrmse = rmse/np.mean(np.abs(gt.iloc[:,idx]))\n",
    "        all_nrmse.append(nrmse)\n",
    "    score = 1.2 * np.sum(all_nrmse[:8]) + 1.0 * np.sum(all_nrmse[8:15]) # Y_01 ~ Y_08 까지 20% 가중치 부여\n",
    "    return score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Y_Feature 개별 NRMSE 계산\n",
    "def lg_individual_nrmse(gt, preds):\n",
    "    \"\"\"\n",
    "    @Description: Metric used in this project (individual)\n",
    "    @Params1: gt, pandas dataframe\n",
    "    @Param2: preds, pandas dataframe\n",
    "    @Return: nrmse score\n",
    "    \"\"\"\n",
    "    rmse = mean_squared_error(gt, preds, squared=False)\n",
    "    nrmse = rmse/np.mean(np.abs(gt))\n",
    "    return nrmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 데이터에서 X_feature, Y_feature 구분\n",
    "def dataset_split_X_y(df):    \n",
    "    \"\"\"\n",
    "    @Description: split data into features and labels\n",
    "    @Param: df, pandas dataframe with columns starting with X for features and Y for labels\n",
    "    @Return: features and labels in pandas dataframes\n",
    "    \"\"\"\n",
    "    xs = df.filter(regex='X') # Input : X Feature\n",
    "    ys = df.filter(regex='Y') # Output : Y Feature\n",
    "    return xs, ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DF 안의 결측치(NA)를 확인\n",
    "def check_for_NAs(df, show=False):\n",
    "    \"\"\"\n",
    "    @Description: checks for the NAs in the dataframe\n",
    "    @Param1: df, pandas dataframe\n",
    "    @Param2: show, boolean indicating whether NaN data are also necessary as a part of the output\n",
    "    @Return: name of the columns with NaN\n",
    "    \"\"\"\n",
    "    nan_values = df.loc[:, df.isnull().any()]\n",
    "    if show:\n",
    "        return df[df.isna().any(axis=1)]\n",
    "    return list(nan_values.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DF 안의 결측치(NA)를 확인\n",
    "def check_for_NAs(df, show=False):\n",
    "    \"\"\"\n",
    "    @Description: checks for the NAs in the dataframe\n",
    "    @Param1: df, pandas dataframe\n",
    "    @Param2: show, boolean indicating whether NaN data are also necessary as a part of the output\n",
    "    @Return: name of the columns with NaN\n",
    "    \"\"\"\n",
    "    nan_values = df.loc[:, df.isnull().any()]\n",
    "    if show:\n",
    "        return df[df.isna().any(axis=1)]\n",
    "    return list(nan_values.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 분산이 0인 feature 탐색\n",
    "def zero_variance(train_x):\n",
    "    \"\"\"\n",
    "    @Description: check for zero_variance\n",
    "    @Param1: df, pandas dataframe\n",
    "    @Return: names of the columns with zero variance\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    for col in train_x.columns:\n",
    "        if train_x[col].var() == 0:\n",
    "            result.append(col)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 가장 높은 상관계수값과 feature 탐색\n",
    "def get_top_correlation(df, n=10):\n",
    "    \"\"\"\n",
    "    @Description: print out top correlated features\n",
    "    @Param1: df, pandas dataframe\n",
    "    @Param2: n, number of lines to print \n",
    "    @Return: pandas series\n",
    "    \"\"\"\n",
    "    pairs = set()\n",
    "    for idx1 in range(0, df.shape[1]):\n",
    "        for idx2 in range(0, idx1+1):\n",
    "            pairs.add((df.columns[idx1], df.columns[idx2]))\n",
    "    corr = df.corr().abs().unstack()\n",
    "    corr = corr.drop(labels=pairs).sort_values(ascending=False)\n",
    "    return corr[0:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 정규화 진행 및 이상치 탐색\n",
    "def find_outlier_zscore(data, threshold = 3):\n",
    "    mean = np.mean(data)\n",
    "    std = np.std(data)\n",
    "    zs = [(y - mean) / std for y in data]\n",
    "    masks = np.where(np.abs(zs) > threshold)\n",
    "    return masks[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# histogram 시각화\n",
    "def adjacent_histogram_boxplot(feature_var, figsize = (7, 5)):\n",
    "    \"\"\"\n",
    "    @Description: plot histogram and boxplot in next to each other\n",
    "    @Param1: feature_var, pandas series \n",
    "    @Param2: figsize, size of the figure \n",
    "    \"\"\"\n",
    "    fig, (hist_plot, box_plot) = plt.subplots(nrows=2, sharex=True, gridspec_kw={'height_ratios':(.85,.15)}, figsize=figsize)\n",
    "    sns.distplot(feature_var, kde=True, ax=hist_plot, kde_kws= {\"linewidth\":1.5}) \n",
    "    sns.boxplot(feature_var, ax=box_plot, linewidth = 1, width = 0.5)\n",
    "    hist_plot.set_ylabel('')    \n",
    "    hist_plot.set_xlabel('')\n",
    "    box_plot.set_xlabel('')\n",
    "    hist_plot.tick_params(labelsize=8)\n",
    "    box_plot.tick_params(labelsize=8)\n",
    "    fig.suptitle(feature_var.name, fontsize = 10)\n",
    "    hist_plot.axvline(np.mean(feature_var),color='red',linestyle='-',lw = 1.5)\n",
    "    hist_plot.axvline(np.median(feature_var),color='green',linestyle='--',lw = 1.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data 불러오기\n",
    "def load_data(train, test):\n",
    "    train_df = pd.read_csv(train)\n",
    "    test_df = pd.read_csv(test)\n",
    "\n",
    "    train_x, train_y = dataset_split_X_y(train_df)\n",
    "    cols_with_zero_variance = zero_variance(train_x) # 분산이 0 (통과 여부)\n",
    "    train_x = train_x.drop(cols_with_zero_variance, axis=1)\n",
    "    \n",
    "    test_df = test_df.drop(cols_with_zero_variance, axis=1)\n",
    "\n",
    "    train_x = train_x.drop(['X_10', 'X_11'], axis = 1) # 결측치가 많음 (결측치 = 0, 공지사항)\n",
    "    test_df = test_df.drop(['X_10', 'X_11'], axis = 1)\n",
    "\n",
    "    test_df = test_df.drop('ID', axis=1) \n",
    "\n",
    "    return train_x, train_y, test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "FB59rtt-llsC"
   },
   "outputs": [],
   "source": [
    "utils.seed_everything(utils.Config.seed)\n",
    "\n",
    "train_df = pd.read_csv('Data/train.csv')\n",
    "test_x = pd.read_csv('Data/test.csv')\n",
    "train_x, train_y = preprocessing.dataset_split_X_y(train_df)\n",
    "\n",
    "cols_with_zero_variance = preprocessing.zero_variance(train_x) # 분산이 0 (통과 여부)\n",
    "train_x = train_x.drop(cols_with_zero_variance, axis = 1)\n",
    "test_x = test_x.drop(cols_with_zero_variance, axis = 1)\n",
    "\n",
    "train_x = train_x.drop(['X_10', 'X_11'], axis = 1) # 결측치가 많음 http://localhost:8888/notebooks/20220801%20LG%20AI%20Research%20%EC%9E%90%EC%9C%A8%EC%A3%BC%ED%96%89%20%EC%84%BC%EC%84%9C%EC%9D%98%20%EC%95%88%ED%85%8C%EB%82%98%20%EC%84%B1%EB%8A%A5%20%EC%98%88%EC%B8%A1%20AI%20%EA%B2%BD%EC%A7%84%EB%8C%80%ED%9A%8C/Model_Submit/Submit1_Stacking%20ensemble_Full_Version.ipynb#(결측치 = 0, 공지사항)\n",
    "test_x = test_x.drop(['X_10', 'X_11'], axis = 1)\n",
    "\n",
    "test_x = test_x.drop('ID', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Y_01 ~ Y_14 반복을 위한 List\n",
    "target = train_y.columns.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Setting\n",
    "def lgbm_objective(params, target):\n",
    "    params = {\n",
    "        'n_estimators': int(params['n_estimators']),\n",
    "        'max_depth': int(params['max_depth']),\n",
    "        'num_leaves': int(params['num_leaves']),\n",
    "        'min_child_samples': int(params['min_child_samples']),\n",
    "        'colsample_bytree': '{:.5f}'.format(params['colsample_bytree']),\n",
    "        'subsample': '{:.5f}'.format(params['subsample']),\n",
    "        'min_split_gain': '{:.5f}'.format(params['min_split_gain']),\n",
    "        'scale_pos_weight': '{:.5f}'.format(params['scale_pos_weight']),\n",
    "        'reg_alpha': '{:.5f}'.format(params['reg_alpha']),\n",
    "        'reg_lambda': '{:.5f}'.format(params['reg_lambda']),\n",
    "        'learning_rate': '{:.5f}'.format(params['learning_rate']),   \n",
    "    }\n",
    "\n",
    "    model = LGBMRegressor(\n",
    "        n_jobs = -1,\n",
    "        random_state = 1,\n",
    "        **params\n",
    "    )\n",
    "\n",
    "    losses = np.sqrt(-cross_val_score(model, train_x, train_y[target], cv=Config.cv, scoring='neg_mean_squared_error')) # cross_val_score : 교차검증\n",
    "    losses = losses / np.mean(np.abs(train_y[target]))\n",
    "    print(\"NRMSE Loss {:.5f} params {}\".format(losses.mean(), params))\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Tunning\n",
    "space_lgbm = {\n",
    "    'n_estimators' : hp.quniform('n_estimators', 10, 30, 5), # 100, 1500,1\n",
    "    'max_depth': hp.quniform('max_depth', 5, 250, 1),\n",
    "    'num_leaves': hp.quniform('num_leaves', 20, 200, 5),\n",
    "    'min_child_samples': hp.quniform('min_child_samples', 10, 150, 5),\n",
    "    'colsample_bytree': hp.uniform('colsample_bytree', 0.3, 1.0),\n",
    "    'subsample': hp.uniform('subsample', 0.3, 1.0),\n",
    "    'min_split_gain': hp.uniform('min_split_gain', 0, 0.7),\n",
    "    'scale_pos_weight': hp.uniform('scale_pos_weight', 1, 10),\n",
    "    'reg_alpha': hp.uniform('reg_alpha', 0, 500),\n",
    "    'reg_lambda': hp.uniform('reg_lambda', 0, 500),\n",
    "    'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(0.5)),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NRMSE Loss 0.26193 params {'n_estimators': 10, 'max_depth': 203, 'num_leaves': 170, 'min_child_samples': 120, 'colsample_bytree': '0.39658', 'subsample': '0.59458', 'min_split_gain': '0.51338', 'scale_pos_weight': '1.65109', 'reg_alpha': '53.98910', 'reg_lambda': '49.41766', 'learning_rate': '0.03674'}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.10s/trial, best loss: 0.2619324646898002]\n",
      "Y_01\n",
      "{'colsample_bytree': 0.39658347909403524, 'learning_rate': 0.03673708428131537, 'max_depth': 203, 'min_child_samples': 120, 'min_split_gain': 0.5133775816035386, 'n_estimators': 10, 'num_leaves': 170, 'reg_alpha': 53.98909751244874, 'reg_lambda': 49.417658170536505, 'scale_pos_weight': 1.6510881557018906, 'subsample': 0.5945769051885113}\n",
      "NRMSE Loss 0.36391 params {'n_estimators': 30, 'max_depth': 241, 'num_leaves': 60, 'min_child_samples': 80, 'colsample_bytree': '0.78958', 'subsample': '0.95996', 'min_split_gain': '0.31844', 'scale_pos_weight': '4.05843', 'reg_alpha': '489.04347', 'reg_lambda': '267.65826', 'learning_rate': '0.17671'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.19s/trial, best loss: 0.36390577290212006]\n",
      "Y_02\n",
      "{'colsample_bytree': 0.7895843024877882, 'learning_rate': 0.17670574555061824, 'max_depth': 241, 'min_child_samples': 80, 'min_split_gain': 0.318439528316908, 'n_estimators': 30, 'num_leaves': 60, 'reg_alpha': 489.04347479282285, 'reg_lambda': 267.6582585186813, 'scale_pos_weight': 4.058427971930797, 'subsample': 0.9599565092838773}\n",
      "NRMSE Loss 0.35548 params {'n_estimators': 30, 'max_depth': 224, 'num_leaves': 170, 'min_child_samples': 20, 'colsample_bytree': '0.88827', 'subsample': '0.77479', 'min_split_gain': '0.60248', 'scale_pos_weight': '6.65480', 'reg_alpha': '141.75957', 'reg_lambda': '105.45166', 'learning_rate': '0.01101'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.34s/trial, best loss: 0.35547808218766985]\n",
      "Y_03\n",
      "{'colsample_bytree': 0.8882655185613018, 'learning_rate': 0.011005718937153625, 'max_depth': 224, 'min_child_samples': 20, 'min_split_gain': 0.6024779401623527, 'n_estimators': 30, 'num_leaves': 170, 'reg_alpha': 141.75957271641798, 'reg_lambda': 105.45165924300525, 'scale_pos_weight': 6.654798868288865, 'subsample': 0.7747866457491239}\n",
      "NRMSE Loss 0.19330 params {'n_estimators': 25, 'max_depth': 39, 'num_leaves': 65, 'min_child_samples': 90, 'colsample_bytree': '0.56805', 'subsample': '0.80849', 'min_split_gain': '0.21617', 'scale_pos_weight': '1.30599', 'reg_alpha': '374.00832', 'reg_lambda': '131.40312', 'learning_rate': '0.10158'}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.64s/trial, best loss: 0.1932974975052124]\n",
      "Y_04\n",
      "{'colsample_bytree': 0.5680529021321612, 'learning_rate': 0.10158225853833021, 'max_depth': 39, 'min_child_samples': 90, 'min_split_gain': 0.21616594570184758, 'n_estimators': 25, 'num_leaves': 65, 'reg_alpha': 374.00832460387727, 'reg_lambda': 131.40311602120397, 'scale_pos_weight': 1.3059948131156314, 'subsample': 0.8084882164592744}\n",
      "NRMSE Loss 0.08061 params {'n_estimators': 20, 'max_depth': 208, 'num_leaves': 195, 'min_child_samples': 140, 'colsample_bytree': '0.75433', 'subsample': '0.35149', 'min_split_gain': '0.57788', 'scale_pos_weight': '7.48821', 'reg_alpha': '3.11884', 'reg_lambda': '185.61876', 'learning_rate': '0.04502'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.15s/trial, best loss: 0.08060786370548952]\n",
      "Y_05\n",
      "{'colsample_bytree': 0.7543299853777504, 'learning_rate': 0.045020302571034884, 'max_depth': 208, 'min_child_samples': 140, 'min_split_gain': 0.5778804435337326, 'n_estimators': 20, 'num_leaves': 195, 'reg_alpha': 3.1188408020705882, 'reg_lambda': 185.6187627793049, 'scale_pos_weight': 7.488208183027353, 'subsample': 0.3514935531042597}\n",
      "NRMSE Loss 0.09312 params {'n_estimators': 25, 'max_depth': 238, 'num_leaves': 35, 'min_child_samples': 105, 'colsample_bytree': '0.96023', 'subsample': '0.45544', 'min_split_gain': '0.03307', 'scale_pos_weight': '9.53773', 'reg_alpha': '368.14658', 'reg_lambda': '19.04816', 'learning_rate': '0.11505'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.48s/trial, best loss: 0.09312290619821714]\n",
      "Y_06\n",
      "{'colsample_bytree': 0.9602267043845085, 'learning_rate': 0.11504620318220009, 'max_depth': 238, 'min_child_samples': 105, 'min_split_gain': 0.033069359946777496, 'n_estimators': 25, 'num_leaves': 35, 'reg_alpha': 368.1465769515348, 'reg_lambda': 19.048160898355814, 'scale_pos_weight': 9.537732255840233, 'subsample': 0.4554388193815837}\n",
      "NRMSE Loss 0.13248 params {'n_estimators': 15, 'max_depth': 227, 'num_leaves': 125, 'min_child_samples': 65, 'colsample_bytree': '0.39729', 'subsample': '0.48769', 'min_split_gain': '0.50753', 'scale_pos_weight': '2.65127', 'reg_alpha': '198.31548', 'reg_lambda': '270.26088', 'learning_rate': '0.02883'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.12s/trial, best loss: 0.13247878604239982]\n",
      "Y_07\n",
      "{'colsample_bytree': 0.3972888936450359, 'learning_rate': 0.028825914620633126, 'max_depth': 227, 'min_child_samples': 65, 'min_split_gain': 0.5075344137825287, 'n_estimators': 15, 'num_leaves': 125, 'reg_alpha': 198.31547835000367, 'reg_lambda': 270.2608840937953, 'scale_pos_weight': 2.651274557261125, 'subsample': 0.48768988289821036}\n",
      "NRMSE Loss 0.02457 params {'n_estimators': 25, 'max_depth': 44, 'num_leaves': 35, 'min_child_samples': 135, 'colsample_bytree': '0.41500', 'subsample': '0.53749', 'min_split_gain': '0.22382', 'scale_pos_weight': '5.52522', 'reg_alpha': '152.72687', 'reg_lambda': '378.35425', 'learning_rate': '0.04797'}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.27s/trial, best loss: 0.024566926169021085]\n",
      "Y_08\n",
      "{'colsample_bytree': 0.41499791897207483, 'learning_rate': 0.0479741510460136, 'max_depth': 44, 'min_child_samples': 135, 'min_split_gain': 0.2238186902555602, 'n_estimators': 25, 'num_leaves': 35, 'reg_alpha': 152.72686805097396, 'reg_lambda': 378.3542535528114, 'scale_pos_weight': 5.525218874433554, 'subsample': 0.537491180395036}\n",
      "NRMSE Loss 0.02472 params {'n_estimators': 20, 'max_depth': 41, 'num_leaves': 175, 'min_child_samples': 55, 'colsample_bytree': '0.31105', 'subsample': '0.33526', 'min_split_gain': '0.48071', 'scale_pos_weight': '7.72493', 'reg_alpha': '177.26208', 'reg_lambda': '25.39656', 'learning_rate': '0.01031'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.22s/trial, best loss: 0.02472416335796536]\n",
      "Y_09\n",
      "{'colsample_bytree': 0.3110543003637211, 'learning_rate': 0.010307928196405951, 'max_depth': 41, 'min_child_samples': 55, 'min_split_gain': 0.48071273795661457, 'n_estimators': 20, 'num_leaves': 175, 'reg_alpha': 177.2620825334969, 'reg_lambda': 25.396558527945867, 'scale_pos_weight': 7.724929989752309, 'subsample': 0.33525540016503474}\n",
      "NRMSE Loss 0.04057 params {'n_estimators': 15, 'max_depth': 103, 'num_leaves': 200, 'min_child_samples': 80, 'colsample_bytree': '0.56349', 'subsample': '0.82279', 'min_split_gain': '0.36281', 'scale_pos_weight': '7.18572', 'reg_alpha': '33.58252', 'reg_lambda': '452.54378', 'learning_rate': '0.01776'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.43s/trial, best loss: 0.04056965769019437]\n",
      "Y_10\n",
      "{'colsample_bytree': 0.563489902775585, 'learning_rate': 0.01775994617160043, 'max_depth': 103, 'min_child_samples': 80, 'min_split_gain': 0.36280537281836006, 'n_estimators': 15, 'num_leaves': 200, 'reg_alpha': 33.58252307735754, 'reg_lambda': 452.54378001927347, 'scale_pos_weight': 7.185716421009021, 'subsample': 0.8227948970547285}\n",
      "NRMSE Loss 0.03390 params {'n_estimators': 30, 'max_depth': 28, 'num_leaves': 85, 'min_child_samples': 90, 'colsample_bytree': '0.72522', 'subsample': '0.38764', 'min_split_gain': '0.03549', 'scale_pos_weight': '9.72601', 'reg_alpha': '53.66328', 'reg_lambda': '79.03338', 'learning_rate': '0.02342'}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.16s/trial, best loss: 0.03390415405131532]\n",
      "Y_11\n",
      "{'colsample_bytree': 0.7252188612362516, 'learning_rate': 0.02341855448016677, 'max_depth': 28, 'min_child_samples': 90, 'min_split_gain': 0.03549053420719814, 'n_estimators': 30, 'num_leaves': 85, 'reg_alpha': 53.663277990315116, 'reg_lambda': 79.03338401636617, 'scale_pos_weight': 9.726007833385838, 'subsample': 0.3876395766723919}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NRMSE Loss 0.02473 params {'n_estimators': 25, 'max_depth': 40, 'num_leaves': 180, 'min_child_samples': 120, 'colsample_bytree': '0.83230', 'subsample': '0.76652', 'min_split_gain': '0.65547', 'scale_pos_weight': '1.91277', 'reg_alpha': '397.92068', 'reg_lambda': '307.19096', 'learning_rate': '0.02474'}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.40s/trial, best loss: 0.024725736754998236]\n",
      "Y_12\n",
      "{'colsample_bytree': 0.8323020179353042, 'learning_rate': 0.024736956232586874, 'max_depth': 40, 'min_child_samples': 120, 'min_split_gain': 0.6554688453585785, 'n_estimators': 25, 'num_leaves': 180, 'reg_alpha': 397.9206823626536, 'reg_lambda': 307.1909564217311, 'scale_pos_weight': 1.9127717216686992, 'subsample': 0.7665181526249765}\n",
      "NRMSE Loss 0.02488 params {'n_estimators': 15, 'max_depth': 68, 'num_leaves': 110, 'min_child_samples': 35, 'colsample_bytree': '0.36902', 'subsample': '0.65841', 'min_split_gain': '0.62838', 'scale_pos_weight': '9.25762', 'reg_alpha': '319.49100', 'reg_lambda': '94.80239', 'learning_rate': '0.01151'}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.20s/trial, best loss: 0.024882593342372566]\n",
      "Y_13\n",
      "{'colsample_bytree': 0.3690238362816472, 'learning_rate': 0.011506880812903915, 'max_depth': 68, 'min_child_samples': 35, 'min_split_gain': 0.6283765923327954, 'n_estimators': 15, 'num_leaves': 110, 'reg_alpha': 319.4910002114118, 'reg_lambda': 94.80238669724056, 'scale_pos_weight': 9.257619763317214, 'subsample': 0.6584104959838084}\n",
      "NRMSE Loss 0.02482 params {'n_estimators': 15, 'max_depth': 49, 'num_leaves': 55, 'min_child_samples': 140, 'colsample_bytree': '0.38373', 'subsample': '0.61391', 'min_split_gain': '0.54421', 'scale_pos_weight': '9.61351', 'reg_alpha': '190.36761', 'reg_lambda': '325.01500', 'learning_rate': '0.02026'}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.22s/trial, best loss: 0.024822466399967602]\n",
      "Y_14\n",
      "{'colsample_bytree': 0.38372672231815713, 'learning_rate': 0.020262291553962326, 'max_depth': 49, 'min_child_samples': 140, 'min_split_gain': 0.5442090350659547, 'n_estimators': 15, 'num_leaves': 55, 'reg_alpha': 190.36761417525605, 'reg_lambda': 325.014997380339, 'scale_pos_weight': 9.613512242142212, 'subsample': 0.613913064417243}\n"
     ]
    }
   ],
   "source": [
    "# Y_01 ~ Y_14 반복\n",
    "\n",
    "best_params_lgbm= []\n",
    "\n",
    "for idx in range(len(target)) :\n",
    "    \n",
    "    lgbm_objective_lambda = lambda params : lgbm_objective(params, target = target[idx])\n",
    "    \n",
    "    best = fmin(fn = lgbm_objective_lambda,\n",
    "            space = space_lgbm,\n",
    "            algo = tpe.suggest,\n",
    "            max_evals = 1) # 200\n",
    "    \n",
    "    \n",
    "    best['n_estimators'] = int(best['n_estimators'])\n",
    "    best['num_leaves'] = int(best['num_leaves'])\n",
    "    best['max_depth'] = int(best['max_depth'])\n",
    "    best['min_child_samples'] = int(best['min_child_samples'])\n",
    "    best_params_lgbm.append(best)\n",
    "    print(target[idx])\n",
    "    print(best)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Catboost Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Setting\n",
    "def cat_objective(params, target):\n",
    "    params = {\n",
    "        'n_estimators': int(params['n_estimators']),\n",
    "        'depth': int(params['depth']),\n",
    "        'learning_rate': params['learning_rate'],   \n",
    "        'l2_leaf_reg': params['l2_leaf_reg'],\n",
    "        'max_bin': int(params['max_bin']),\n",
    "        'min_data_in_leaf': int(params['min_data_in_leaf']),\n",
    "        'random_strength': params['random_strength'],\n",
    "        'fold_len_multiplier': params['fold_len_multiplier'],\n",
    "        \n",
    "    }\n",
    "\n",
    "    model = CatBoostRegressor(\n",
    "        logging_level='Silent',\n",
    "        **params\n",
    "    )\n",
    "\n",
    "    losses = np.sqrt(-cross_val_score(model, train_x, train_y[target], cv=Config.cv, scoring='neg_mean_squared_error'))\n",
    "    losses = losses / np.mean(np.abs(train_y[target]))\n",
    "    print(\"NRMSE Loss {:.5f} params {}\".format(losses.mean(), params))\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Tunning\n",
    "space_catboost = {\n",
    "    'n_estimators' : hp.quniform('n_estimators', 10, 20, 5), #100, 300, 50\n",
    "    'depth': hp.quniform(\"depth\", 2, 16, 1),\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'l2_leaf_reg': hp.uniform('l2_leaf_reg', 3, 8),\n",
    "    'max_bin' : hp.quniform('max_bin', 1, 254, 1),\n",
    "    'min_data_in_leaf' : hp.quniform('min_data_in_leaf', 2, 700, 1),\n",
    "    'random_strength' : hp.loguniform('random_strength', np.log(0.005), np.log(5)),\n",
    "    'fold_len_multiplier' : hp.loguniform('fold_len_multiplier', np.log(1.01), np.log(2.5)),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NRMSE Loss 0.25774 params {'n_estimators': 20, 'depth': 3, 'learning_rate': 0.1901721693222083, 'l2_leaf_reg': 3.5202853420904185, 'max_bin': 128, 'min_data_in_leaf': 400, 'random_strength': 0.015078229232325525, 'fold_len_multiplier': 1.3446311829393718}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.69s/trial, best loss: 0.25773876062886714]\n",
      "Y_01\n",
      "{'depth': 3.0, 'fold_len_multiplier': 1.3446311829393718, 'l2_leaf_reg': 3.5202853420904185, 'learning_rate': 0.1901721693222083, 'max_bin': 128.0, 'min_data_in_leaf': 400.0, 'n_estimators': 20.0, 'random_strength': 0.015078229232325525}\n",
      "NRMSE Loss 0.35981 params {'n_estimators': 15, 'depth': 4, 'learning_rate': 0.16883024869452154, 'l2_leaf_reg': 6.607539761800004, 'max_bin': 102, 'min_data_in_leaf': 534, 'random_strength': 2.45096006131398, 'fold_len_multiplier': 1.1898791666291697}\n",
      "100%|████████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.58s/trial, best loss: 0.35981310462888]\n",
      "Y_02\n",
      "{'depth': 4.0, 'fold_len_multiplier': 1.1898791666291697, 'l2_leaf_reg': 6.607539761800004, 'learning_rate': 0.16883024869452154, 'max_bin': 102.0, 'min_data_in_leaf': 534.0, 'n_estimators': 15.0, 'random_strength': 2.45096006131398}\n",
      "NRMSE Loss 0.35014 params {'n_estimators': 20, 'depth': 9, 'learning_rate': 0.13394713589084753, 'l2_leaf_reg': 3.1722923613732545, 'max_bin': 49, 'min_data_in_leaf': 74, 'random_strength': 0.4929672781772541, 'fold_len_multiplier': 1.4606334855918017}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.89s/trial, best loss: 0.3501428455010481]\n",
      "Y_03\n",
      "{'depth': 9.0, 'fold_len_multiplier': 1.4606334855918017, 'l2_leaf_reg': 3.1722923613732545, 'learning_rate': 0.13394713589084753, 'max_bin': 49.0, 'min_data_in_leaf': 74.0, 'n_estimators': 20.0, 'random_strength': 0.4929672781772541}\n",
      "NRMSE Loss 0.19501 params {'n_estimators': 10, 'depth': 4, 'learning_rate': 0.1670742299327343, 'l2_leaf_reg': 3.8581150509102633, 'max_bin': 211, 'min_data_in_leaf': 138, 'random_strength': 3.697236979643192, 'fold_len_multiplier': 2.3262787909082885}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.40s/trial, best loss: 0.1950111456148472]\n",
      "Y_04\n",
      "{'depth': 4.0, 'fold_len_multiplier': 2.3262787909082885, 'l2_leaf_reg': 3.8581150509102633, 'learning_rate': 0.1670742299327343, 'max_bin': 211.0, 'min_data_in_leaf': 138.0, 'n_estimators': 10.0, 'random_strength': 3.697236979643192}\n",
      "NRMSE Loss 0.08072 params {'n_estimators': 15, 'depth': 6, 'learning_rate': 0.09151840737755425, 'l2_leaf_reg': 4.070555186039057, 'max_bin': 81, 'min_data_in_leaf': 132, 'random_strength': 0.25141083957261245, 'fold_len_multiplier': 1.1580092230526555}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.75s/trial, best loss: 0.08071971868307296]\n",
      "Y_05\n",
      "{'depth': 6.0, 'fold_len_multiplier': 1.1580092230526555, 'l2_leaf_reg': 4.070555186039057, 'learning_rate': 0.09151840737755425, 'max_bin': 81.0, 'min_data_in_leaf': 132.0, 'n_estimators': 15.0, 'random_strength': 0.25141083957261245}\n",
      "NRMSE Loss 0.09360 params {'n_estimators': 15, 'depth': 7, 'learning_rate': 0.048671445327476304, 'l2_leaf_reg': 5.285958503935586, 'max_bin': 147, 'min_data_in_leaf': 573, 'random_strength': 0.11265057061060706, 'fold_len_multiplier': 1.2130474528261783}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.05s/trial, best loss: 0.09360373263821811]\n",
      "Y_06\n",
      "{'depth': 7.0, 'fold_len_multiplier': 1.2130474528261783, 'l2_leaf_reg': 5.285958503935586, 'learning_rate': 0.048671445327476304, 'max_bin': 147.0, 'min_data_in_leaf': 573.0, 'n_estimators': 15.0, 'random_strength': 0.11265057061060706}\n",
      "NRMSE Loss 0.13152 params {'n_estimators': 15, 'depth': 7, 'learning_rate': 0.05192294338057758, 'l2_leaf_reg': 6.569725755369169, 'max_bin': 116, 'min_data_in_leaf': 599, 'random_strength': 0.03711275678486101, 'fold_len_multiplier': 1.7530860564857023}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.01s/trial, best loss: 0.13152043327023968]\n",
      "Y_07\n",
      "{'depth': 7.0, 'fold_len_multiplier': 1.7530860564857023, 'l2_leaf_reg': 6.569725755369169, 'learning_rate': 0.05192294338057758, 'max_bin': 116.0, 'min_data_in_leaf': 599.0, 'n_estimators': 15.0, 'random_strength': 0.03711275678486101}\n",
      "NRMSE Loss 0.02422 params {'n_estimators': 10, 'depth': 11, 'learning_rate': 0.18598513079252352, 'l2_leaf_reg': 3.700744447521667, 'max_bin': 35, 'min_data_in_leaf': 504, 'random_strength': 2.2441064588653794, 'fold_len_multiplier': 1.8518961837335135}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:03<00:00,  3.41s/trial, best loss: 0.024223740814432942]\n",
      "Y_08\n",
      "{'depth': 11.0, 'fold_len_multiplier': 1.8518961837335135, 'l2_leaf_reg': 3.700744447521667, 'learning_rate': 0.18598513079252352, 'max_bin': 35.0, 'min_data_in_leaf': 504.0, 'n_estimators': 10.0, 'random_strength': 2.2441064588653794}\n",
      "NRMSE Loss 0.02408 params {'n_estimators': 15, 'depth': 14, 'learning_rate': 0.18168093580105585, 'l2_leaf_reg': 6.314253978748072, 'max_bin': 128, 'min_data_in_leaf': 443, 'random_strength': 0.07975793819775964, 'fold_len_multiplier': 1.5434783141733839}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:59<00:00, 59.74s/trial, best loss: 0.024084755712537417]\n",
      "Y_09\n",
      "{'depth': 14.0, 'fold_len_multiplier': 1.5434783141733839, 'l2_leaf_reg': 6.314253978748072, 'learning_rate': 0.18168093580105585, 'max_bin': 128.0, 'min_data_in_leaf': 443.0, 'n_estimators': 15.0, 'random_strength': 0.07975793819775964}\n",
      "NRMSE Loss 0.03931 params {'n_estimators': 15, 'depth': 4, 'learning_rate': 0.1937122502080459, 'l2_leaf_reg': 7.41598859672694, 'max_bin': 215, 'min_data_in_leaf': 151, 'random_strength': 1.9806002804900993, 'fold_len_multiplier': 1.5379311998038234}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.74s/trial, best loss: 0.03930872125351016]\n",
      "Y_10\n",
      "{'depth': 4.0, 'fold_len_multiplier': 1.5379311998038234, 'l2_leaf_reg': 7.41598859672694, 'learning_rate': 0.1937122502080459, 'max_bin': 215.0, 'min_data_in_leaf': 151.0, 'n_estimators': 15.0, 'random_strength': 1.9806002804900993}\n",
      "NRMSE Loss 0.03373 params {'n_estimators': 10, 'depth': 13, 'learning_rate': 0.17839762145383448, 'l2_leaf_reg': 4.497896773562145, 'max_bin': 221, 'min_data_in_leaf': 529, 'random_strength': 0.17525026863618873, 'fold_len_multiplier': 1.512857690573662}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:28<00:00, 28.43s/trial, best loss: 0.03372512069605764]\n",
      "Y_11\n",
      "{'depth': 13.0, 'fold_len_multiplier': 1.512857690573662, 'l2_leaf_reg': 4.497896773562145, 'learning_rate': 0.17839762145383448, 'max_bin': 221.0, 'min_data_in_leaf': 529.0, 'n_estimators': 10.0, 'random_strength': 0.17525026863618873}\n",
      "NRMSE Loss 0.02410 params {'n_estimators': 20, 'depth': 11, 'learning_rate': 0.2941312149543491, 'l2_leaf_reg': 3.6857396925673642, 'max_bin': 44, 'min_data_in_leaf': 56, 'random_strength': 0.03845735078222035, 'fold_len_multiplier': 1.841689196505443}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:06<00:00,  6.43s/trial, best loss: 0.02409828268854989]\n",
      "Y_12\n",
      "{'depth': 11.0, 'fold_len_multiplier': 1.841689196505443, 'l2_leaf_reg': 3.6857396925673642, 'learning_rate': 0.2941312149543491, 'max_bin': 44.0, 'min_data_in_leaf': 56.0, 'n_estimators': 20.0, 'random_strength': 0.03845735078222035}\n",
      "NRMSE Loss 0.02437 params {'n_estimators': 15, 'depth': 5, 'learning_rate': 0.05557293005298978, 'l2_leaf_reg': 5.705578070648929, 'max_bin': 81, 'min_data_in_leaf': 324, 'random_strength': 1.0512010200755604, 'fold_len_multiplier': 1.337475038070316}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.67s/trial, best loss: 0.024369507372901007]\n",
      "Y_13\n",
      "{'depth': 5.0, 'fold_len_multiplier': 1.337475038070316, 'l2_leaf_reg': 5.705578070648929, 'learning_rate': 0.05557293005298978, 'max_bin': 81.0, 'min_data_in_leaf': 324.0, 'n_estimators': 15.0, 'random_strength': 1.0512010200755604}\n",
      "NRMSE Loss 0.02440 params {'n_estimators': 15, 'depth': 5, 'learning_rate': 0.04789080066265784, 'l2_leaf_reg': 4.248006763440869, 'max_bin': 175, 'min_data_in_leaf': 502, 'random_strength': 0.009171366406988821, 'fold_len_multiplier': 1.8978300118431357}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.77s/trial, best loss: 0.024403933061224188]\n",
      "Y_14\n",
      "{'depth': 5.0, 'fold_len_multiplier': 1.8978300118431357, 'l2_leaf_reg': 4.248006763440869, 'learning_rate': 0.04789080066265784, 'max_bin': 175.0, 'min_data_in_leaf': 502.0, 'n_estimators': 15.0, 'random_strength': 0.009171366406988821}\n"
     ]
    }
   ],
   "source": [
    "# Y_01 ~ Y_14 반복\n",
    "\n",
    "best_params_cat= []\n",
    "\n",
    "for idx in range(len(target)) :\n",
    "    \n",
    "    cat_objective_lambda = lambda params : cat_objective(params, target = target[idx])\n",
    "    \n",
    "    best = fmin(fn = cat_objective_lambda,\n",
    "            space = space_catboost,\n",
    "            algo = tpe.suggest,\n",
    "            verbose = 100,\n",
    "            max_evals = 1)\n",
    "    \n",
    "    best_params_cat.append(best)\n",
    "    print(target[idx])\n",
    "    print(best)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extra Tree Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Y_01 ~ Y_14 반복을 위한 List\n",
    "target = ['Y_01', 'Y_02', 'Y_03','Y_04','Y_05','Y_06','Y_07','Y_08','Y_09','Y_10','Y_11','Y_12','Y_13','Y_14']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Setting\n",
    "def extra_objective(params, target):\n",
    "    params = {\n",
    "        'n_estimators': int(params['n_estimators']),\n",
    "        'max_depth': int(params['max_depth']),\n",
    "        #'min_samples_split': int(params['min_samples_split']),\n",
    "        'min_samples_leaf': int(params['min_samples_leaf']),\n",
    "        'min_weight_fraction_leaf': params['min_weight_fraction_leaf'],\n",
    "        'max_features': params['max_features'],\n",
    "        'max_leaf_nodes': int(params['max_leaf_nodes']),\n",
    "        'min_impurity_decrease': params['min_impurity_decrease'],\n",
    "        'bootstrap': params['bootstrap'],\n",
    "        'ccp_alpha': params['ccp_alpha'],  \n",
    "    }\n",
    "\n",
    "    model = ExtraTreesRegressor(\n",
    "        n_jobs = -1,\n",
    "        verbose = 0,\n",
    "        random_state = 1,\n",
    "        **params\n",
    "    )\n",
    "\n",
    "    losses = np.sqrt(-cross_val_score(model, train_x, train_y[target], cv=Config.cv, scoring='neg_mean_squared_error'))\n",
    "    losses = losses / np.mean(np.abs(train_y[target]))\n",
    "    print(\"NRMSE Loss {:.5f} params {}\".format(losses.mean(), params))\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Tunning\n",
    "space_extra = {\n",
    "    'n_estimators' : hp.quniform('n_estimators', 10, 20, 5), # 100, 1500, 50\n",
    "    'max_depth': hp.quniform('max_depth', 3, 50, 1),\n",
    "    #'min_samples_split': hp.quniform('min_samples_split', 0.5, 1, 0.5), # 1 이하여야 함\n",
    "    'min_samples_leaf': hp.quniform('min_samples_leaf', 5, 50, 1),\n",
    "    'min_weight_fraction_leaf': hp.uniform('min_weight_fraction_leaf', 0.01, 0.5),\n",
    "    'max_features': hp.choice('max_features', ['sqrt', 'log2', None, 'auto']),\n",
    "    'max_leaf_nodes': hp.quniform('max_leaf_nodes', 3, 30, 1),\n",
    "    'min_impurity_decrease': hp.uniform('min_impurity_decrease', 0, 200),\n",
    "    'bootstrap':  hp.choice('bootstrap', [True, False]),\n",
    "    'ccp_alpha': hp.uniform('ccp_alpha', 0.01, 1.0),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NRMSE Loss 0.26302 params {'n_estimators': 10, 'max_depth': 6, 'min_samples_leaf': 11, 'min_weight_fraction_leaf': 0.3301713104021617, 'max_features': 'log2', 'max_leaf_nodes': 16, 'min_impurity_decrease': 5.086342801287125, 'bootstrap': True, 'ccp_alpha': 0.562204083307845}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:06<00:00,  6.28s/trial, best loss: 0.26301810737744125]\n",
      "Y_01\n",
      "{'bootstrap': 0, 'ccp_alpha': 0.562204083307845, 'max_depth': 6, 'max_features': 1, 'max_leaf_nodes': 16, 'min_impurity_decrease': 5.086342801287125, 'min_samples_leaf': 11.0, 'min_weight_fraction_leaf': 0.3301713104021617, 'n_estimators': 10}\n",
      "NRMSE Loss 0.36548 params {'n_estimators': 15, 'max_depth': 29, 'min_samples_leaf': 21, 'min_weight_fraction_leaf': 0.2742311968427292, 'max_features': 'auto', 'max_leaf_nodes': 28, 'min_impurity_decrease': 68.41529138540776, 'bootstrap': False, 'ccp_alpha': 0.14102518464702204}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.88s/trial, best loss: 0.36547870413392214]\n",
      "Y_02\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.14102518464702204, 'max_depth': 29, 'max_features': 3, 'max_leaf_nodes': 28, 'min_impurity_decrease': 68.41529138540776, 'min_samples_leaf': 21.0, 'min_weight_fraction_leaf': 0.2742311968427292, 'n_estimators': 15}\n",
      "NRMSE Loss 0.35644 params {'n_estimators': 10, 'max_depth': 30, 'min_samples_leaf': 28, 'min_weight_fraction_leaf': 0.1099388242348514, 'max_features': 'log2', 'max_leaf_nodes': 9, 'min_impurity_decrease': 125.47666201879568, 'bootstrap': False, 'ccp_alpha': 0.14751911885204783}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.59s/trial, best loss: 0.3564399973350015]\n",
      "Y_03\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.14751911885204783, 'max_depth': 30, 'max_features': 1, 'max_leaf_nodes': 9, 'min_impurity_decrease': 125.47666201879568, 'min_samples_leaf': 28.0, 'min_weight_fraction_leaf': 0.1099388242348514, 'n_estimators': 10}\n",
      "NRMSE Loss 0.19730 params {'n_estimators': 15, 'max_depth': 39, 'min_samples_leaf': 46, 'min_weight_fraction_leaf': 0.3006847464399027, 'max_features': None, 'max_leaf_nodes': 12, 'min_impurity_decrease': 76.02229438571048, 'bootstrap': False, 'ccp_alpha': 0.7989414596324305}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.88s/trial, best loss: 0.19729775673738048]\n",
      "Y_04\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.7989414596324305, 'max_depth': 39, 'max_features': 2, 'max_leaf_nodes': 12, 'min_impurity_decrease': 76.02229438571048, 'min_samples_leaf': 46.0, 'min_weight_fraction_leaf': 0.3006847464399027, 'n_estimators': 15}\n",
      "NRMSE Loss 0.08124 params {'n_estimators': 15, 'max_depth': 42, 'min_samples_leaf': 5, 'min_weight_fraction_leaf': 0.44071780758257595, 'max_features': 'auto', 'max_leaf_nodes': 21, 'min_impurity_decrease': 13.600652015551074, 'bootstrap': False, 'ccp_alpha': 0.4565371404319639}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.93s/trial, best loss: 0.08123990393519516]\n",
      "Y_05\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.4565371404319639, 'max_depth': 42, 'max_features': 3, 'max_leaf_nodes': 21, 'min_impurity_decrease': 13.600652015551074, 'min_samples_leaf': 5.0, 'min_weight_fraction_leaf': 0.44071780758257595, 'n_estimators': 15}\n",
      "NRMSE Loss 0.09610 params {'n_estimators': 15, 'max_depth': 49, 'min_samples_leaf': 47, 'min_weight_fraction_leaf': 0.24444842465859148, 'max_features': 'sqrt', 'max_leaf_nodes': 27, 'min_impurity_decrease': 66.06796830568572, 'bootstrap': True, 'ccp_alpha': 0.961076745341051}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.76s/trial, best loss: 0.09610078645875328]\n",
      "Y_06\n",
      "{'bootstrap': 0, 'ccp_alpha': 0.961076745341051, 'max_depth': 49, 'max_features': 0, 'max_leaf_nodes': 27, 'min_impurity_decrease': 66.06796830568572, 'min_samples_leaf': 47.0, 'min_weight_fraction_leaf': 0.24444842465859148, 'n_estimators': 15}\n",
      "NRMSE Loss 0.13274 params {'n_estimators': 15, 'max_depth': 17, 'min_samples_leaf': 40, 'min_weight_fraction_leaf': 0.37086961102964827, 'max_features': 'log2', 'max_leaf_nodes': 12, 'min_impurity_decrease': 3.0788823829743572, 'bootstrap': False, 'ccp_alpha': 0.14982564013181926}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.64s/trial, best loss: 0.1327385864525858]\n",
      "Y_07\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.14982564013181926, 'max_depth': 17, 'max_features': 1, 'max_leaf_nodes': 12, 'min_impurity_decrease': 3.0788823829743572, 'min_samples_leaf': 40.0, 'min_weight_fraction_leaf': 0.37086961102964827, 'n_estimators': 15}\n",
      "NRMSE Loss 0.02513 params {'n_estimators': 20, 'max_depth': 30, 'min_samples_leaf': 32, 'min_weight_fraction_leaf': 0.19403085567528294, 'max_features': 'auto', 'max_leaf_nodes': 28, 'min_impurity_decrease': 184.32345989068017, 'bootstrap': False, 'ccp_alpha': 0.9322065130060772}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.03s/trial, best loss: 0.025133500588735237]\n",
      "Y_08\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.9322065130060772, 'max_depth': 30, 'max_features': 3, 'max_leaf_nodes': 28, 'min_impurity_decrease': 184.32345989068017, 'min_samples_leaf': 32.0, 'min_weight_fraction_leaf': 0.19403085567528294, 'n_estimators': 20}\n",
      "NRMSE Loss 0.02485 params {'n_estimators': 15, 'max_depth': 5, 'min_samples_leaf': 44, 'min_weight_fraction_leaf': 0.03193165577301049, 'max_features': None, 'max_leaf_nodes': 19, 'min_impurity_decrease': 54.00127144963758, 'bootstrap': True, 'ccp_alpha': 0.8948268586781886}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.86s/trial, best loss: 0.024847869218610347]\n",
      "Y_09\n",
      "{'bootstrap': 0, 'ccp_alpha': 0.8948268586781886, 'max_depth': 5, 'max_features': 2, 'max_leaf_nodes': 19, 'min_impurity_decrease': 54.00127144963758, 'min_samples_leaf': 44.0, 'min_weight_fraction_leaf': 0.03193165577301049, 'n_estimators': 15}\n",
      "NRMSE Loss 0.04107 params {'n_estimators': 15, 'max_depth': 11, 'min_samples_leaf': 31, 'min_weight_fraction_leaf': 0.4805460311550347, 'max_features': 'auto', 'max_leaf_nodes': 27, 'min_impurity_decrease': 67.35872402356556, 'bootstrap': True, 'ccp_alpha': 0.46328079181490195}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.87s/trial, best loss: 0.041068478937856014]\n",
      "Y_10\n",
      "{'bootstrap': 0, 'ccp_alpha': 0.46328079181490195, 'max_depth': 11, 'max_features': 3, 'max_leaf_nodes': 27, 'min_impurity_decrease': 67.35872402356556, 'min_samples_leaf': 31.0, 'min_weight_fraction_leaf': 0.4805460311550347, 'n_estimators': 15}\n",
      "NRMSE Loss 0.03411 params {'n_estimators': 10, 'max_depth': 42, 'min_samples_leaf': 36, 'min_weight_fraction_leaf': 0.18624273732144295, 'max_features': 'sqrt', 'max_leaf_nodes': 23, 'min_impurity_decrease': 182.51923846903694, 'bootstrap': False, 'ccp_alpha': 0.5076615726613026}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.63s/trial, best loss: 0.03411380454982242]\n",
      "Y_11\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.5076615726613026, 'max_depth': 42, 'max_features': 0, 'max_leaf_nodes': 23, 'min_impurity_decrease': 182.51923846903694, 'min_samples_leaf': 36.0, 'min_weight_fraction_leaf': 0.18624273732144295, 'n_estimators': 10}\n",
      "NRMSE Loss 0.02503 params {'n_estimators': 15, 'max_depth': 47, 'min_samples_leaf': 6, 'min_weight_fraction_leaf': 0.42447463288474935, 'max_features': 'log2', 'max_leaf_nodes': 11, 'min_impurity_decrease': 167.17131916798604, 'bootstrap': True, 'ccp_alpha': 0.3575831200917303}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.68s/trial, best loss: 0.025025315064817255]\n",
      "Y_12\n",
      "{'bootstrap': 0, 'ccp_alpha': 0.3575831200917303, 'max_depth': 47, 'max_features': 1, 'max_leaf_nodes': 11, 'min_impurity_decrease': 167.17131916798604, 'min_samples_leaf': 6.0, 'min_weight_fraction_leaf': 0.42447463288474935, 'n_estimators': 15}\n",
      "NRMSE Loss 0.02498 params {'n_estimators': 15, 'max_depth': 11, 'min_samples_leaf': 9, 'min_weight_fraction_leaf': 0.07041991083746443, 'max_features': 'auto', 'max_leaf_nodes': 18, 'min_impurity_decrease': 26.189493967836363, 'bootstrap': False, 'ccp_alpha': 0.9257024580701733}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.86s/trial, best loss: 0.02497875953096713]\n",
      "Y_13\n",
      "{'bootstrap': 1, 'ccp_alpha': 0.9257024580701733, 'max_depth': 11, 'max_features': 3, 'max_leaf_nodes': 18, 'min_impurity_decrease': 26.189493967836363, 'min_samples_leaf': 9.0, 'min_weight_fraction_leaf': 0.07041991083746443, 'n_estimators': 15}\n",
      "NRMSE Loss 0.02500 params {'n_estimators': 15, 'max_depth': 37, 'min_samples_leaf': 21, 'min_weight_fraction_leaf': 0.1706274283513835, 'max_features': None, 'max_leaf_nodes': 7, 'min_impurity_decrease': 88.19158265380365, 'bootstrap': True, 'ccp_alpha': 0.6049504118308293}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.89s/trial, best loss: 0.024997457562067973]\n",
      "Y_14\n",
      "{'bootstrap': 0, 'ccp_alpha': 0.6049504118308293, 'max_depth': 37, 'max_features': 2, 'max_leaf_nodes': 7, 'min_impurity_decrease': 88.19158265380365, 'min_samples_leaf': 21.0, 'min_weight_fraction_leaf': 0.1706274283513835, 'n_estimators': 15}\n"
     ]
    }
   ],
   "source": [
    "# Y_01 ~ Y_14 반복\n",
    "\n",
    "best_params_extra= []\n",
    "\n",
    "for idx in range(len(target)) :\n",
    "    \n",
    "    extra_objective_lambda = lambda params : extra_objective(params, target = target[idx])\n",
    "    \n",
    "    best = fmin(fn = extra_objective_lambda,\n",
    "            space = space_extra,\n",
    "            algo = tpe.suggest,\n",
    "            verbose = 100,\n",
    "            max_evals = 1) #200\n",
    "    \n",
    "    best['n_estimators'] = int(best['n_estimators'])\n",
    "    best['max_depth'] = int(best['max_depth'])\n",
    "    best['max_leaf_nodes'] = int(best['max_leaf_nodes'])\n",
    "    best_params_extra.append(best)\n",
    "    print(target[idx])\n",
    "    print(best)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NGBR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Setting\n",
    "def ngbr_objective(params, target):\n",
    "    params = {\n",
    "        'n_estimators': int(params['n_estimators']),\n",
    "        'learning_rate': params['learning_rate'],\n",
    "        'natural_gradient': params['natural_gradient'],\n",
    "        'col_sample': float(params['col_sample']),\n",
    "        'minibatch_frac': float(params['minibatch_frac']),\n",
    "        'tol': float(params['tol']),\n",
    "    }\n",
    "\n",
    "    model = NGBRegressor(\n",
    "        verbose = 100,\n",
    "        random_state = 1,\n",
    "        **params\n",
    "    )\n",
    "\n",
    "    losses = np.sqrt(-cross_val_score(model, train_x, train_y[target], cv=Config.cv, scoring='neg_mean_squared_error'))\n",
    "    losses = losses / np.mean(np.abs(train_y[target]))\n",
    "    print(\"NRMSE Loss {:.5f} params {}\".format(losses.mean(), params))\n",
    "    return losses.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameter Tunning\n",
    "space_ngboost = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 10, 30, 10), #100, 500, 10\n",
    "    'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),\n",
    "    'natural_gradient': hp.choice('natural_gradient', [True, False]),\n",
    "    'col_sample': hp.quniform('col_sample', 0, 1, 0.01),\n",
    "    'minibatch_frac': hp.quniform('minibatch_frac', 0, 1, 0.01),\n",
    "    'tol': hp.uniform('tol', 1e-6, 3e-4),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[iter 0] loss=0.3836 val_loss=0.0000 scale=0.2500 norm=0.6399                                                          \n",
      "[iter 0] loss=0.3873 val_loss=0.0000 scale=0.2500 norm=0.6449                                                          \n",
      "[iter 0] loss=0.3999 val_loss=0.0000 scale=0.2500 norm=0.6455                                                          \n",
      "[iter 0] loss=0.4002 val_loss=0.0000 scale=0.2500 norm=0.6443                                                          \n",
      "[iter 0] loss=0.3965 val_loss=0.0000 scale=0.2500 norm=0.6409                                                          \n",
      "[iter 0] loss=0.4000 val_loss=0.0000 scale=0.2500 norm=0.6388                                                          \n",
      "[iter 0] loss=0.3893 val_loss=0.0000 scale=0.2500 norm=0.6395                                                          \n",
      "[iter 0] loss=0.3865 val_loss=0.0000 scale=0.2500 norm=0.6364                                                          \n",
      "[iter 0] loss=0.3918 val_loss=0.0000 scale=0.2500 norm=0.6360                                                          \n",
      "[iter 0] loss=0.3930 val_loss=0.0000 scale=0.2500 norm=0.6356                                                          \n",
      "NRMSE Loss 0.25874 params {'n_estimators': 20, 'learning_rate': 0.2254601901288807, 'natural_gradient': False, 'col_sample': 0.33, 'minibatch_frac': 0.4, 'tol': 0.0001987423960805551}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:14<00:00, 14.61s/trial, best loss: 0.25873566169238194]\n",
      "Y_01\n",
      "{'col_sample': 0.33, 'learning_rate': 0.2254601901288807, 'minibatch_frac': 0.4, 'n_estimators': 20, 'natural_gradient': 1, 'tol': 0.0001987423960805551}\n",
      "[iter 0] loss=0.4658 val_loss=0.0000 scale=0.2500 norm=0.6042                                                          \n",
      "[iter 0] loss=0.4650 val_loss=0.0000 scale=0.2500 norm=0.6050                                                          \n",
      "[iter 0] loss=0.4715 val_loss=0.0000 scale=0.2500 norm=0.6016                                                          \n",
      "[iter 0] loss=0.4703 val_loss=0.0000 scale=0.2500 norm=0.6030                                                          \n",
      "[iter 0] loss=0.4695 val_loss=0.0000 scale=0.2500 norm=0.6035                                                          \n",
      "[iter 0] loss=0.4722 val_loss=0.0000 scale=0.2500 norm=0.6022                                                          \n",
      "[iter 0] loss=0.4692 val_loss=0.0000 scale=0.2500 norm=0.6029                                                          \n",
      "[iter 0] loss=0.4662 val_loss=0.0000 scale=0.2500 norm=0.6031                                                          \n",
      "[iter 0] loss=0.4713 val_loss=0.0000 scale=0.2500 norm=0.6016                                                          \n",
      "[iter 0] loss=0.4701 val_loss=0.0000 scale=0.2500 norm=0.6026                                                          \n",
      "NRMSE Loss 0.36168 params {'n_estimators': 20, 'learning_rate': 0.03250451450389757, 'natural_gradient': False, 'col_sample': 0.21, 'minibatch_frac': 0.91, 'tol': 0.00029924539691337853}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:20<00:00, 20.93s/trial, best loss: 0.36168220948861246]\n",
      "Y_02\n",
      "{'col_sample': 0.21, 'learning_rate': 0.03250451450389757, 'minibatch_frac': 0.91, 'n_estimators': 20, 'natural_gradient': 1, 'tol': 0.00029924539691337853}\n",
      "[iter 0] loss=0.4080 val_loss=0.0000 scale=1.0000 norm=0.6142                                                          \n",
      "[iter 0] loss=0.4082 val_loss=0.0000 scale=1.0000 norm=0.6169                                                          \n",
      "[iter 0] loss=0.4190 val_loss=0.0000 scale=1.0000 norm=0.6229                                                          \n",
      "[iter 0] loss=0.4163 val_loss=0.0000 scale=1.0000 norm=0.6225                                                          \n",
      "[iter 0] loss=0.4066 val_loss=0.0000 scale=1.0000 norm=0.6188                                                          \n",
      "[iter 0] loss=0.4122 val_loss=0.0000 scale=1.0000 norm=0.6205                                                          \n",
      "[iter 0] loss=0.4040 val_loss=0.0000 scale=1.0000 norm=0.6156                                                          \n",
      "[iter 0] loss=0.4018 val_loss=0.0000 scale=1.0000 norm=0.6156                                                          \n",
      "[iter 0] loss=0.4060 val_loss=0.0000 scale=1.0000 norm=0.6162                                                          \n",
      "[iter 0] loss=0.4058 val_loss=0.0000 scale=1.0000 norm=0.6154                                                          \n",
      "NRMSE Loss 0.35228 params {'n_estimators': 10, 'learning_rate': 0.2339385807759383, 'natural_gradient': True, 'col_sample': 0.37, 'minibatch_frac': 0.36, 'tol': 8.160658272493064e-05}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:07<00:00,  7.87s/trial, best loss: 0.3522821042042671]\n",
      "Y_03\n",
      "{'col_sample': 0.37, 'learning_rate': 0.2339385807759383, 'minibatch_frac': 0.36, 'n_estimators': 10, 'natural_gradient': 0, 'tol': 8.160658272493064e-05}\n",
      "[iter 0] loss=2.3773 val_loss=0.0000 scale=1.0000 norm=2.2246                                                          \n",
      "[iter 0] loss=2.3849 val_loss=0.0000 scale=1.0000 norm=2.2412                                                          \n",
      "[iter 0] loss=2.3805 val_loss=0.0000 scale=1.0000 norm=2.2287                                                          \n",
      "[iter 0] loss=2.3865 val_loss=0.0000 scale=1.0000 norm=2.2423                                                          \n",
      "[iter 0] loss=2.3827 val_loss=0.0000 scale=1.0000 norm=2.2369                                                          \n",
      "[iter 0] loss=2.3896 val_loss=0.0000 scale=1.0000 norm=2.2524                                                          \n",
      "[iter 0] loss=2.3852 val_loss=0.0000 scale=1.0000 norm=2.2446                                                          \n",
      "[iter 0] loss=2.3895 val_loss=0.0000 scale=1.0000 norm=2.2536                                                          \n",
      "[iter 0] loss=2.4049 val_loss=0.0000 scale=1.0000 norm=2.2879                                                          \n",
      "[iter 0] loss=2.4026 val_loss=0.0000 scale=1.0000 norm=2.2803                                                          \n",
      "NRMSE Loss 0.19780 params {'n_estimators': 30, 'learning_rate': 0.268334750351281, 'natural_gradient': True, 'col_sample': 0.68, 'minibatch_frac': 0.16, 'tol': 0.00012032901073923133}\n",
      "100%|███████████████████████████████████████████████████| 1/1 [00:20<00:00, 20.31s/trial, best loss: 0.197798845921081]\n",
      "Y_04\n",
      "{'col_sample': 0.68, 'learning_rate': 0.268334750351281, 'minibatch_frac': 0.16, 'n_estimators': 30, 'natural_gradient': 0, 'tol': 0.00012032901073923133}\n",
      "[iter 0] loss=2.3294 val_loss=0.0000 scale=1.0000 norm=2.1095                                                          \n",
      "[iter 0] loss=2.3413 val_loss=0.0000 scale=1.0000 norm=2.1329                                                          \n",
      "[iter 0] loss=2.3382 val_loss=0.0000 scale=1.0000 norm=2.1265                                                          \n",
      "[iter 0] loss=2.3410 val_loss=0.0000 scale=1.0000 norm=2.1290                                                          \n",
      "[iter 0] loss=2.3382 val_loss=0.0000 scale=1.0000 norm=2.1267                                                          \n",
      "[iter 0] loss=2.3443 val_loss=0.0000 scale=1.0000 norm=2.1428                                                          \n",
      "[iter 0] loss=2.3401 val_loss=0.0000 scale=1.0000 norm=2.1357                                                          \n",
      "[iter 0] loss=2.3270 val_loss=0.0000 scale=1.0000 norm=2.1060                                                          \n",
      "[iter 0] loss=2.3463 val_loss=0.0000 scale=1.0000 norm=2.1428                                                          \n",
      "[iter 0] loss=2.3500 val_loss=0.0000 scale=1.0000 norm=2.1436                                                          \n",
      "NRMSE Loss 0.08101 params {'n_estimators': 10, 'learning_rate': 0.11497876173453539, 'natural_gradient': True, 'col_sample': 0.42, 'minibatch_frac': 0.18, 'tol': 1.2104516645576657e-05}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:05<00:00,  5.25s/trial, best loss: 0.08101238589076347]\n",
      "Y_05\n",
      "{'col_sample': 0.42, 'learning_rate': 0.11497876173453539, 'minibatch_frac': 0.18, 'n_estimators': 10, 'natural_gradient': 0, 'tol': 1.2104516645576657e-05}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[iter 0] loss=1.9588 val_loss=0.0000 scale=1.0000 norm=1.6477                                                          \n",
      "[iter 0] loss=1.9400 val_loss=0.0000 scale=0.5000 norm=0.8370                                                          \n",
      "[iter 0] loss=2.0129 val_loss=0.0000 scale=0.5000 norm=0.8591                                                          \n",
      "[iter 0] loss=2.0499 val_loss=0.0000 scale=0.5000 norm=0.8705                                                          \n",
      "[iter 0] loss=2.1251 val_loss=0.0000 scale=1.0000 norm=1.7705                                                          \n",
      "[iter 0] loss=2.1251 val_loss=0.0000 scale=1.0000 norm=1.7688                                                          \n",
      "[iter 0] loss=2.1241 val_loss=0.0000 scale=1.0000 norm=1.7719                                                          \n",
      "[iter 0] loss=2.1243 val_loss=0.0000 scale=1.0000 norm=1.7722                                                          \n",
      "[iter 0] loss=2.1254 val_loss=0.0000 scale=1.0000 norm=1.7681                                                          \n",
      "[iter 0] loss=2.1248 val_loss=0.0000 scale=1.0000 norm=1.7695                                                          \n",
      "NRMSE Loss 0.09568 params {'n_estimators': 20, 'learning_rate': 0.1529959726958761, 'natural_gradient': False, 'col_sample': 0.46, 'minibatch_frac': 0.8300000000000001, 'tol': 0.0002814261213752231}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:37<00:00, 37.64s/trial, best loss: 0.09567829904011868]\n",
      "Y_06\n",
      "{'col_sample': 0.46, 'learning_rate': 0.1529959726958761, 'minibatch_frac': 0.8300000000000001, 'n_estimators': 20, 'natural_gradient': 1, 'tol': 0.0002814261213752231}\n",
      "[iter 0] loss=0.5273 val_loss=0.0000 scale=0.2500 norm=0.5660                                                          \n",
      "[iter 0] loss=0.5415 val_loss=0.0000 scale=0.2500 norm=0.5721                                                          \n",
      "[iter 0] loss=0.5418 val_loss=0.0000 scale=0.2500 norm=0.5703                                                          \n",
      "[iter 0] loss=0.5377 val_loss=0.0000 scale=0.2500 norm=0.5674                                                          \n",
      "[iter 0] loss=0.5485 val_loss=0.0000 scale=0.2500 norm=0.5653                                                          \n",
      "[iter 0] loss=0.5595 val_loss=0.0000 scale=0.2500 norm=0.5651                                                          \n",
      "[iter 0] loss=0.5552 val_loss=0.0000 scale=0.2500 norm=0.5672                                                          \n",
      "[iter 0] loss=0.5510 val_loss=0.0000 scale=0.2500 norm=0.5705                                                          \n",
      "[iter 0] loss=0.5604 val_loss=0.0000 scale=0.2500 norm=0.5668                                                          \n",
      "[iter 0] loss=0.5630 val_loss=0.0000 scale=0.2500 norm=0.5675                                                          \n",
      "NRMSE Loss 0.13216 params {'n_estimators': 10, 'learning_rate': 0.045777483827373816, 'natural_gradient': False, 'col_sample': 0.22, 'minibatch_frac': 0.18, 'tol': 3.453426689850043e-05}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:03<00:00,  3.33s/trial, best loss: 0.1321584842372249]\n",
      "Y_07\n",
      "{'col_sample': 0.22, 'learning_rate': 0.045777483827373816, 'minibatch_frac': 0.18, 'n_estimators': 10, 'natural_gradient': 1, 'tol': 3.453426689850043e-05}\n",
      "[iter 0] loss=0.9881 val_loss=0.0000 scale=0.5000 norm=0.8514                                                          \n",
      "[iter 0] loss=0.9984 val_loss=0.0000 scale=0.5000 norm=0.8485                                                          \n",
      "[iter 0] loss=0.9982 val_loss=0.0000 scale=0.5000 norm=0.8488                                                          \n",
      "[iter 0] loss=1.0032 val_loss=0.0000 scale=0.5000 norm=0.8470                                                          \n",
      "[iter 0] loss=1.0096 val_loss=0.0000 scale=0.5000 norm=0.8444                                                          \n",
      "[iter 0] loss=1.0126 val_loss=0.0000 scale=0.5000 norm=0.8403                                                          \n",
      "[iter 0] loss=1.0052 val_loss=0.0000 scale=0.5000 norm=0.8448                                                          \n",
      "[iter 0] loss=1.0081 val_loss=0.0000 scale=0.5000 norm=0.8450                                                          \n",
      "[iter 0] loss=1.0171 val_loss=0.0000 scale=0.5000 norm=0.8400                                                          \n",
      "[iter 0] loss=1.0113 val_loss=0.0000 scale=0.5000 norm=0.8393                                                          \n",
      "NRMSE Loss 0.02423 params {'n_estimators': 20, 'learning_rate': 0.09938820976881206, 'natural_gradient': False, 'col_sample': 0.52, 'minibatch_frac': 0.72, 'tol': 5.3617628120681205e-05}\n",
      "100%|████████████████████████████████████████████████| 1/1 [00:36<00:00, 36.80s/trial, best loss: 0.024227078343365998]\n",
      "Y_08\n",
      "{'col_sample': 0.52, 'learning_rate': 0.09938820976881206, 'minibatch_frac': 0.72, 'n_estimators': 20, 'natural_gradient': 1, 'tol': 5.3617628120681205e-05}\n",
      "[iter 0] loss=0.9748 val_loss=0.0000 scale=1.0000 norm=0.7724                                                          \n",
      "[iter 0] loss=0.9861 val_loss=0.0000 scale=1.0000 norm=0.7781                                                          \n",
      "[iter 0] loss=0.9866 val_loss=0.0000 scale=1.0000 norm=0.7792                                                          \n",
      "[iter 0] loss=0.9929 val_loss=0.0000 scale=1.0000 norm=0.7825                                                          \n",
      "[iter 0] loss=0.9995 val_loss=0.0000 scale=1.0000 norm=0.7847                                                          \n",
      "[iter 0] loss=1.0013 val_loss=0.0000 scale=1.0000 norm=0.7833                                                          \n",
      "[iter 0] loss=0.9950 val_loss=0.0000 scale=1.0000 norm=0.7813                                                          \n",
      "[iter 0] loss=0.9956 val_loss=0.0000 scale=1.0000 norm=0.7808                                                          \n",
      "[iter 0] loss=1.0037 val_loss=0.0000 scale=1.0000 norm=0.7833                                                          \n",
      "[iter 0] loss=1.0009 val_loss=0.0000 scale=1.0000 norm=0.7822                                                          \n",
      "NRMSE Loss 0.02398 params {'n_estimators': 20, 'learning_rate': 0.27651431960414935, 'natural_gradient': True, 'col_sample': 0.45, 'minibatch_frac': 0.58, 'tol': 0.00021179128966041096}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:27<00:00, 27.55s/trial, best loss: 0.02397554216605453]\n",
      "Y_09\n",
      "{'col_sample': 0.45, 'learning_rate': 0.27651431960414935, 'minibatch_frac': 0.58, 'n_estimators': 20, 'natural_gradient': 0, 'tol': 0.00021179128966041096}\n",
      "[iter 0] loss=1.3080 val_loss=0.0000 scale=1.0000 norm=1.4378                                                          \n",
      "[iter 0] loss=1.3351 val_loss=0.0000 scale=1.0000 norm=1.4923                                                          \n",
      "[iter 0] loss=1.3508 val_loss=0.0000 scale=1.0000 norm=1.5010                                                          \n",
      "[iter 0] loss=1.3524 val_loss=0.0000 scale=1.0000 norm=1.4930                                                          \n",
      "[iter 0] loss=1.3564 val_loss=0.0000 scale=1.0000 norm=1.5026                                                          \n",
      "[iter 0] loss=1.3588 val_loss=0.0000 scale=1.0000 norm=1.4978                                                          \n",
      "[iter 0] loss=1.3521 val_loss=0.0000 scale=1.0000 norm=1.5040                                                          \n",
      "[iter 0] loss=1.3537 val_loss=0.0000 scale=1.0000 norm=1.5003                                                          \n",
      "[iter 0] loss=1.3622 val_loss=0.0000 scale=1.0000 norm=1.4961                                                          \n",
      "[iter 0] loss=1.3597 val_loss=0.0000 scale=1.0000 norm=1.5014                                                          \n",
      "NRMSE Loss 0.03968 params {'n_estimators': 10, 'learning_rate': 0.18575424412705777, 'natural_gradient': False, 'col_sample': 0.25, 'minibatch_frac': 0.45, 'tol': 8.375247761925755e-05}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████| 1/1 [00:06<00:00,  6.71s/trial, best loss: 0.03968090179749164]\n",
      "Y_10\n",
      "{'col_sample': 0.25, 'learning_rate': 0.18575424412705777, 'minibatch_frac': 0.45, 'n_estimators': 10, 'natural_gradient': 1, 'tol': 8.375247761925755e-05}\n",
      "[iter 0] loss=1.2248 val_loss=0.0000 scale=1.0000 norm=1.5014                                                          \n",
      "[iter 0] loss=1.2342 val_loss=0.0000 scale=1.0000 norm=1.4935                                                          \n",
      "[iter 0] loss=1.2324 val_loss=0.0000 scale=1.0000 norm=1.4954                                                          \n",
      "[iter 0] loss=1.2322 val_loss=0.0000 scale=1.0000 norm=1.4947                                                          \n",
      "[iter 0] loss=1.2353 val_loss=0.0000 scale=1.0000 norm=1.4919                                                          \n",
      "[iter 0] loss=1.2381 val_loss=0.0000 scale=1.0000 norm=1.4871                                                          \n",
      "[iter 0] loss=1.2303 val_loss=0.0000 scale=1.0000 norm=1.4931                                                          \n",
      "[iter 0] loss=1.2298 val_loss=0.0000 scale=1.0000 norm=1.4942                                                          \n",
      "[iter 0] loss=1.2399 val_loss=0.0000 scale=1.0000 norm=1.4853                                                          \n",
      "[iter 0] loss=1.2374 val_loss=0.0000 scale=1.0000 norm=1.4881                                                          \n",
      "NRMSE Loss 0.03372 params {'n_estimators': 30, 'learning_rate': 0.2993327011092276, 'natural_gradient': False, 'col_sample': 0.19, 'minibatch_frac': 0.96, 'tol': 0.00027114834413915356}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:31<00:00, 31.38s/trial, best loss: 0.03372161991638324]\n",
      "Y_11\n",
      "{'col_sample': 0.19, 'learning_rate': 0.2993327011092276, 'minibatch_frac': 0.96, 'n_estimators': 30, 'natural_gradient': 1, 'tol': 0.00027114834413915356}\n",
      "[iter 0] loss=0.9810 val_loss=0.0000 scale=0.5000 norm=0.8523                                                          \n",
      "[iter 0] loss=0.9900 val_loss=0.0000 scale=0.5000 norm=0.8501                                                          \n",
      "[iter 0] loss=0.9894 val_loss=0.0000 scale=0.5000 norm=0.8493                                                          \n",
      "[iter 0] loss=0.9946 val_loss=0.0000 scale=0.5000 norm=0.8474                                                          \n",
      "[iter 0] loss=1.0009 val_loss=0.0000 scale=0.5000 norm=0.8449                                                          \n",
      "[iter 0] loss=1.0031 val_loss=0.0000 scale=0.5000 norm=0.8419                                                          \n",
      "[iter 0] loss=0.9976 val_loss=0.0000 scale=0.5000 norm=0.8464                                                          \n",
      "[iter 0] loss=1.0020 val_loss=0.0000 scale=0.5000 norm=0.8484                                                          \n",
      "[iter 0] loss=1.0102 val_loss=0.0000 scale=0.5000 norm=0.8429                                                          \n",
      "[iter 0] loss=1.0054 val_loss=0.0000 scale=0.5000 norm=0.8429                                                          \n",
      "NRMSE Loss 0.02427 params {'n_estimators': 10, 'learning_rate': 0.13349086269203708, 'natural_gradient': False, 'col_sample': 0.86, 'minibatch_frac': 0.73, 'tol': 0.00018783012555452473}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:30<00:00, 30.06s/trial, best loss: 0.02427257607173148]\n",
      "Y_12\n",
      "{'col_sample': 0.86, 'learning_rate': 0.13349086269203708, 'minibatch_frac': 0.73, 'n_estimators': 10, 'natural_gradient': 1, 'tol': 0.00018783012555452473}\n",
      "[iter 0] loss=0.9701 val_loss=0.0000 scale=0.5000 norm=0.8471                                                          \n",
      "[iter 0] loss=0.9779 val_loss=0.0000 scale=0.5000 norm=0.8439                                                          \n",
      "[iter 0] loss=0.9785 val_loss=0.0000 scale=0.5000 norm=0.8434                                                          \n",
      "[iter 0] loss=0.9854 val_loss=0.0000 scale=0.5000 norm=0.8424                                                          \n",
      "[iter 0] loss=0.9880 val_loss=0.0000 scale=0.5000 norm=0.8366                                                          \n",
      "[iter 0] loss=0.9927 val_loss=0.0000 scale=0.5000 norm=0.8349                                                          \n",
      "[iter 0] loss=0.9863 val_loss=0.0000 scale=0.5000 norm=0.8387                                                          \n",
      "[iter 0] loss=0.9922 val_loss=0.0000 scale=0.5000 norm=0.8422                                                          \n",
      "[iter 0] loss=0.9982 val_loss=0.0000 scale=0.5000 norm=0.8341                                                          \n",
      "[iter 0] loss=0.9957 val_loss=0.0000 scale=0.5000 norm=0.8359                                                          \n",
      "NRMSE Loss 0.02409 params {'n_estimators': 20, 'learning_rate': 0.15565362729338053, 'natural_gradient': False, 'col_sample': 0.5, 'minibatch_frac': 0.28, 'tol': 5.6058983746622924e-05}\n",
      "100%|██████████████████████████████████████████████████| 1/1 [00:15<00:00, 15.49s/trial, best loss: 0.0240891503837612]\n",
      "Y_13\n",
      "{'col_sample': 0.5, 'learning_rate': 0.15565362729338053, 'minibatch_frac': 0.28, 'n_estimators': 20, 'natural_gradient': 1, 'tol': 5.6058983746622924e-05}\n",
      "[iter 0] loss=0.9812 val_loss=0.0000 scale=1.0000 norm=0.7760                                                          \n",
      "[iter 0] loss=0.9780 val_loss=0.0000 scale=1.0000 norm=0.7695                                                          \n",
      "[iter 0] loss=0.9700 val_loss=0.0000 scale=1.0000 norm=0.7622                                                          \n",
      "[iter 0] loss=0.9738 val_loss=0.0000 scale=1.0000 norm=0.7627                                                          \n",
      "[iter 0] loss=0.9842 val_loss=0.0000 scale=1.0000 norm=0.7703                                                          \n",
      "[iter 0] loss=0.9827 val_loss=0.0000 scale=1.0000 norm=0.7637                                                          \n",
      "[iter 0] loss=0.9796 val_loss=0.0000 scale=1.0000 norm=0.7651                                                          \n",
      "[iter 0] loss=1.0112 val_loss=0.0000 scale=1.0000 norm=0.7895                                                          \n",
      "[iter 0] loss=1.0131 val_loss=0.0000 scale=1.0000 norm=0.7866                                                          \n",
      "[iter 0] loss=1.0069 val_loss=0.0000 scale=1.0000 norm=0.7843                                                          \n",
      "NRMSE Loss 0.02432 params {'n_estimators': 20, 'learning_rate': 0.08768286111187894, 'natural_gradient': True, 'col_sample': 0.93, 'minibatch_frac': 0.06, 'tol': 1.3520369176488492e-05}\n",
      "100%|█████████████████████████████████████████████████| 1/1 [00:09<00:00,  9.90s/trial, best loss: 0.02431953209342711]\n",
      "Y_14\n",
      "{'col_sample': 0.93, 'learning_rate': 0.08768286111187894, 'minibatch_frac': 0.06, 'n_estimators': 20, 'natural_gradient': 0, 'tol': 1.3520369176488492e-05}\n"
     ]
    }
   ],
   "source": [
    "# Y_01 ~ Y_14 반복\n",
    "\n",
    "best_params_ngbr= []\n",
    "\n",
    "for idx in range(len(target)) :\n",
    "    \n",
    "    ngbr_objective_lambda = lambda params : ngbr_objective(params, target = target[idx])\n",
    "    \n",
    "    best = fmin(fn = ngbr_objective_lambda,\n",
    "            space = space_ngboost,\n",
    "            algo = tpe.suggest,\n",
    "            verbose = 1000,#10\n",
    "            max_evals = 1) #200\n",
    "    \n",
    "    best['n_estimators'] = int(best['n_estimators'])\n",
    "    best_params_ngbr.append(best)\n",
    "    print(target[idx])\n",
    "    print(best)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking Ensenble\n",
    "- 모델별 개별학습(타겟 Y_01~Y_14) 반복"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Y_01 ~ Y_14 반복을 위한 List\n",
    "target = train_y.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_stacking_base_datasets(model, train_x, train_y, col,test, params):\n",
    "    kf = KFold(n_splits=2, shuffle=False)\n",
    "    train_fold_pred = np.zeros((train_x.shape[0],1))\n",
    "    test_pred = np.zeros((test.shape[0],10))\n",
    "    \n",
    "    \n",
    "    for folder_counter, (train_index, valid_index) in enumerate(kf.split(train_x)):\n",
    "        print('Fold : ', folder_counter, ' Start')\n",
    "        X_tr = train_x.loc[train_index]\n",
    "        y_tr = train_y[col].loc[train_index]\n",
    "        X_te = train_x.loc[valid_index] \n",
    "        \n",
    "        if model == 'cat':\n",
    "          model = CatBoostRegressor(random_state=1,\n",
    "                                    **params)\n",
    "        \n",
    "        elif model == 'extra':\n",
    "          model = ExtraTreesRegressor(random_state=1, \n",
    "                                      **params)\n",
    "\n",
    "        elif model == 'ngbr':\n",
    "          model = NGBRegressor(random_state = 1)\n",
    "        \n",
    "        elif model == 'lgbm':\n",
    "          model = LGBMRegressor(random_state=1, n_jobs=-1, \n",
    "                                **params)\n",
    "\n",
    "        model.fit(X_tr, y_tr)\n",
    "        train_fold_pred[valid_index, :] = model.predict(X_te).reshape(-1,1) \n",
    "        test_pred[:, folder_counter] = model.predict(test) \n",
    "        \n",
    "    test_pred_mean = np.mean(test_pred, axis=1).reshape(-1,1)\n",
    "    \n",
    "    return train_fold_pred, test_pred_mean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Outside target:  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "Before target :  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "index : 0\n",
      "After target:  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "Fold :  0  Start\n",
      "Fold :  1  Start\n",
      "Fold :  0  Start\n",
      "[iter 0] loss=0.3653 val_loss=0.0000 scale=1.0000 norm=0.6084\n",
      "[iter 100] loss=0.3336 val_loss=0.0000 scale=1.0000 norm=0.5934\n",
      "[iter 200] loss=0.3180 val_loss=0.0000 scale=1.0000 norm=0.5931\n",
      "[iter 300] loss=0.3082 val_loss=0.0000 scale=1.0000 norm=0.5924\n",
      "[iter 400] loss=0.3000 val_loss=0.0000 scale=2.0000 norm=1.1830\n",
      "Fold :  1  Start\n",
      "[iter 0] loss=0.4208 val_loss=0.0000 scale=1.0000 norm=0.6925\n",
      "[iter 100] loss=0.3824 val_loss=0.0000 scale=1.0000 norm=0.6345\n",
      "[iter 200] loss=0.3697 val_loss=0.0000 scale=1.0000 norm=0.6211\n",
      "[iter 300] loss=0.3608 val_loss=0.0000 scale=2.0000 norm=1.2322\n",
      "[iter 400] loss=0.3534 val_loss=0.0000 scale=1.0000 norm=0.6138\n",
      "Fold :  0  Start\n",
      "Fold :  1  Start\n",
      "Before target :  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "index : 1\n",
      "After target:  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "Fold :  0  Start\n",
      "Fold :  1  Start\n",
      "Fold :  0  Start\n",
      "[iter 0] loss=0.4599 val_loss=0.0000 scale=1.0000 norm=0.6159\n",
      "[iter 100] loss=0.4342 val_loss=0.0000 scale=1.0000 norm=0.6035\n",
      "[iter 200] loss=0.4213 val_loss=0.0000 scale=1.0000 norm=0.6012\n",
      "[iter 300] loss=0.4119 val_loss=0.0000 scale=2.0000 norm=1.2018\n",
      "[iter 400] loss=0.4045 val_loss=0.0000 scale=1.0000 norm=0.6001\n",
      "Fold :  1  Start\n",
      "[iter 0] loss=0.4750 val_loss=0.0000 scale=1.0000 norm=0.6606\n",
      "[iter 100] loss=0.4475 val_loss=0.0000 scale=1.0000 norm=0.6257\n",
      "[iter 200] loss=0.4369 val_loss=0.0000 scale=2.0000 norm=1.2351\n",
      "[iter 300] loss=0.4286 val_loss=0.0000 scale=1.0000 norm=0.6141\n",
      "[iter 400] loss=0.4222 val_loss=0.0000 scale=1.0000 norm=0.6122\n",
      "Fold :  0  Start\n",
      "Fold :  1  Start\n",
      "Before target :  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "index : 2\n",
      "After target:  ['Y_01', 'Y_02', 'Y_03', 'Y_04', 'Y_05', 'Y_06', 'Y_07', 'Y_08', 'Y_09', 'Y_10', 'Y_11', 'Y_12', 'Y_13', 'Y_14']\n",
      "Fold :  0  Start\n",
      "Fold :  1  Start\n",
      "Fold :  0  Start\n",
      "[iter 0] loss=0.4030 val_loss=0.0000 scale=1.0000 norm=0.6074\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [236]\u001b[0m, in \u001b[0;36m<cell line: 4>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[38;5;66;03m# Y_01 ~ Y_14의 best parameter 가져오기\u001b[39;00m\n\u001b[0;32m     11\u001b[0m \u001b[38;5;66;03m# xx_train, xx_test = get_stacking_base_datasets('cat', train_x, train_y, col=target[idx[, test=test_x, params = best_params_cat[idx])\u001b[39;00m\n\u001b[0;32m     12\u001b[0m zz_train, zz_test \u001b[38;5;241m=\u001b[39m get_stacking_base_datasets(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlgbm\u001b[39m\u001b[38;5;124m'\u001b[39m, train_x, train_y, col\u001b[38;5;241m=\u001b[39mtarget[idx], test\u001b[38;5;241m=\u001b[39mtest_x, params \u001b[38;5;241m=\u001b[39m best_params_lgbm[idx])\n\u001b[1;32m---> 13\u001b[0m qq_train, qq_test \u001b[38;5;241m=\u001b[39m \u001b[43mget_stacking_base_datasets\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mngbr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_x\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain_y\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcol\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtarget\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtest\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtest_x\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mbest_params_ngbr\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     14\u001b[0m yy_train, yy_test \u001b[38;5;241m=\u001b[39m get_stacking_base_datasets(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mextra\u001b[39m\u001b[38;5;124m'\u001b[39m, train_x, train_y, col\u001b[38;5;241m=\u001b[39mtarget[idx], test\u001b[38;5;241m=\u001b[39mtest_x, params \u001b[38;5;241m=\u001b[39m best_params_extra[idx])\n\u001b[0;32m     16\u001b[0m Stack_final_X_train \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mconcatenate((xx_train, yy_train, zz_train, qq_train), axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "Input \u001b[1;32mIn [229]\u001b[0m, in \u001b[0;36mget_stacking_base_datasets\u001b[1;34m(model, train_x, train_y, col, test, params)\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m model \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlgbm\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m     25\u001b[0m   model \u001b[38;5;241m=\u001b[39m LGBMRegressor(random_state\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, n_jobs\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, \n\u001b[0;32m     26\u001b[0m                         \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mparams)\n\u001b[1;32m---> 28\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_tr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_tr\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     29\u001b[0m train_fold_pred[valid_index, :] \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict(X_te)\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m1\u001b[39m) \n\u001b[0;32m     30\u001b[0m test_pred[:, folder_counter] \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict(test) \n",
      "File \u001b[1;32mC:\\Dev\\miniconda3\\lib\\site-packages\\ngboost\\ngboost.py:277\u001b[0m, in \u001b[0;36mNGBoost.fit\u001b[1;34m(self, X, Y, X_val, Y_val, sample_weight, val_sample_weight, train_loss_monitor, val_loss_monitor, early_stopping_rounds)\u001b[0m\n\u001b[0;32m    274\u001b[0m loss \u001b[38;5;241m=\u001b[39m loss_list[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[0;32m    275\u001b[0m grads \u001b[38;5;241m=\u001b[39m D\u001b[38;5;241m.\u001b[39mgrad(Y_batch, natural\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnatural_gradient)\n\u001b[1;32m--> 277\u001b[0m proj_grad \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_base\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX_batch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight_batch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    278\u001b[0m scale \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mline_search(proj_grad, P_batch, Y_batch, weight_batch)\n\u001b[0;32m    280\u001b[0m \u001b[38;5;66;03m# pdb.set_trace()\u001b[39;00m\n",
      "File \u001b[1;32mC:\\Dev\\miniconda3\\lib\\site-packages\\ngboost\\ngboost.py:152\u001b[0m, in \u001b[0;36mNGBoost.fit_base\u001b[1;34m(self, X, grads, sample_weight)\u001b[0m\n\u001b[0;32m    151\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit_base\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, grads, sample_weight\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m--> 152\u001b[0m     models \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m    153\u001b[0m         clone(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mBase)\u001b[38;5;241m.\u001b[39mfit(X, g, sample_weight\u001b[38;5;241m=\u001b[39msample_weight) \u001b[38;5;28;01mfor\u001b[39;00m g \u001b[38;5;129;01min\u001b[39;00m grads\u001b[38;5;241m.\u001b[39mT\n\u001b[0;32m    154\u001b[0m     ]\n\u001b[0;32m    155\u001b[0m     fitted \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray([m\u001b[38;5;241m.\u001b[39mpredict(X) \u001b[38;5;28;01mfor\u001b[39;00m m \u001b[38;5;129;01min\u001b[39;00m models])\u001b[38;5;241m.\u001b[39mT\n\u001b[0;32m    156\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_models\u001b[38;5;241m.\u001b[39mappend(models)\n",
      "File \u001b[1;32mC:\\Dev\\miniconda3\\lib\\site-packages\\ngboost\\ngboost.py:153\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    151\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit_base\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, grads, sample_weight\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m    152\u001b[0m     models \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m--> 153\u001b[0m         \u001b[43mclone\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mBase\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m g \u001b[38;5;129;01min\u001b[39;00m grads\u001b[38;5;241m.\u001b[39mT\n\u001b[0;32m    154\u001b[0m     ]\n\u001b[0;32m    155\u001b[0m     fitted \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray([m\u001b[38;5;241m.\u001b[39mpredict(X) \u001b[38;5;28;01mfor\u001b[39;00m m \u001b[38;5;129;01min\u001b[39;00m models])\u001b[38;5;241m.\u001b[39mT\n\u001b[0;32m    156\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbase_models\u001b[38;5;241m.\u001b[39mappend(models)\n",
      "File \u001b[1;32mC:\\Dev\\miniconda3\\lib\\site-packages\\sklearn\\tree\\_classes.py:1342\u001b[0m, in \u001b[0;36mDecisionTreeRegressor.fit\u001b[1;34m(self, X, y, sample_weight, check_input)\u001b[0m\n\u001b[0;32m   1313\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfit\u001b[39m(\u001b[38;5;28mself\u001b[39m, X, y, sample_weight\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, check_input\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m):\n\u001b[0;32m   1314\u001b[0m     \u001b[38;5;124;03m\"\"\"Build a decision tree regressor from the training set (X, y).\u001b[39;00m\n\u001b[0;32m   1315\u001b[0m \n\u001b[0;32m   1316\u001b[0m \u001b[38;5;124;03m    Parameters\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1339\u001b[0m \u001b[38;5;124;03m        Fitted estimator.\u001b[39;00m\n\u001b[0;32m   1340\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 1342\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1343\u001b[0m \u001b[43m        \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1344\u001b[0m \u001b[43m        \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1345\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1346\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcheck_input\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcheck_input\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1347\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1348\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "File \u001b[1;32mC:\\Dev\\miniconda3\\lib\\site-packages\\sklearn\\tree\\_classes.py:458\u001b[0m, in \u001b[0;36mBaseDecisionTree.fit\u001b[1;34m(self, X, y, sample_weight, check_input)\u001b[0m\n\u001b[0;32m    447\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    448\u001b[0m     builder \u001b[38;5;241m=\u001b[39m BestFirstTreeBuilder(\n\u001b[0;32m    449\u001b[0m         splitter,\n\u001b[0;32m    450\u001b[0m         min_samples_split,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    455\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmin_impurity_decrease,\n\u001b[0;32m    456\u001b[0m     )\n\u001b[1;32m--> 458\u001b[0m \u001b[43mbuilder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuild\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtree_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    460\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_outputs_ \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m is_classifier(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    461\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_classes_ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mn_classes_[\u001b[38;5;241m0\u001b[39m]\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Y_01 ~ Y_14 반복\n",
    "target = train_y.columns.tolist()\n",
    "print('Outside target: ', target)\n",
    "for idx in range(len(target)) :\n",
    "    print('Before target : ', target)\n",
    "    print('index :', idx)\n",
    "    print('After target: ', target)\n",
    "    stack_final = []\n",
    "    \n",
    "    # Y_01 ~ Y_14의 best parameter 가져오기\n",
    "    # xx_train, xx_test = get_stacking_base_datasets('cat', train_x, train_y, col=target[idx[, test=test_x, params = best_params_cat[idx])\n",
    "    zz_train, zz_test = get_stacking_base_datasets('lgbm', train_x, train_y, col=target[idx], test=test_x, params = best_params_lgbm[idx])\n",
    "    qq_train, qq_test = get_stacking_base_datasets('ngbr', train_x, train_y, col=target[idx], test=test_x, params = best_params_ngbr[idx])\n",
    "    yy_train, yy_test = get_stacking_base_datasets('extra', train_x, train_y, col=target[idx], test=test_x, params = best_params_extra[idx])\n",
    "    \n",
    "    Stack_final_X_train = np.concatenate((xx_train, yy_train, zz_train, qq_train), axis=1)\n",
    "    Stack_final_X_test = np.concatenate((xx_test, yy_test, zz_test, qq_test), axis=1)\n",
    "    \n",
    "    # final model 선택\n",
    "    lr_final = LGBMRegressor(**best_params_lgbm[idx])\n",
    "    lr_final.fit(Stack_final_X_train, train_y[target[idx]])\n",
    "    stack_final_csv = lr_final.predict(Stack_final_X_test)\n",
    "    stack_final.append(stack_final_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('Data/sample_submission.csv')\n",
    "\n",
    "for idx in range(len(target)) :\n",
    "    \n",
    "    sub[target[idx]] = stack_final[idx]\n",
    "\n",
    "sub.to_csv('Data/stack_Ensenble.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Save(to CSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5DRbK4etsHG0"
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('Data/sample_submission.csv')\n",
    "sub['Y_01'] = stack_final1\n",
    "sub['Y_02'] = stack_final2\n",
    "sub['Y_03'] = stack_final3\n",
    "sub['Y_04'] = stack_final4\n",
    "sub['Y_05'] = stack_final5\n",
    "sub['Y_06'] = stack_final6\n",
    "sub['Y_07'] = stack_final7\n",
    "sub['Y_08'] = stack_final8\n",
    "sub['Y_09'] = stack_final9\n",
    "sub['Y_10'] = stack_final10\n",
    "sub['Y_11'] = stack_final11\n",
    "sub['Y_12'] = stack_final12\n",
    "sub['Y_13'] = stack_final13\n",
    "sub['Y_14'] = stack_final14\n",
    "sub.to_csv('Data/stack_Ensenble.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "REAL FINAL.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}