MCA_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Importing necessary data science libraries"
      ],
      "metadata": {
        "id": "eu_qrW623qy7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "-297FQcP0HJv"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "from IPython.display import display"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Metrics and Encoder Libraries"
      ],
      "metadata": {
        "id": "x67_APFR3plF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.metrics import make_scorer\n",
        "from sklearn.metrics import mean_pinball_loss, mean_squared_error\n",
        "from sklearn import preprocessing\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score, ConfusionMatrixDisplay, confusion_matrix\n",
        "from sklearn.preprocessing import MinMaxScaler,StandardScaler\n",
        "from sklearn.preprocessing import LabelEncoder, OneHotEncoder"
      ],
      "metadata": {
        "id": "p0Bx2ZeI3iTQ"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Regression Models"
      ],
      "metadata": {
        "id": "HjpS5VV64N0E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import GridSearchCV, StratifiedKFold, learning_curve\n",
        "from sklearn.neighbors import KNeighborsRegressor\n",
        "from sklearn.neural_network import MLPRegressor\n",
        "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor, VotingRegressor, AdaBoostRegressor\n",
        "from sklearn.tree import DecisionTreeRegressor\n",
        "from sklearn.ensemble import GradientBoostingRegressor\n",
        "from sklearn.linear_model import ElasticNet\n",
        "from sklearn.linear_model import SGDRegressor\n",
        "from sklearn.svm import SVR\n",
        "from sklearn.linear_model import BayesianRidge\n",
        "from sklearn.kernel_ridge import KernelRidge\n",
        "from sklearn.linear_model import LinearRegression\n",
        "from xgboost.sklearn import XGBRegressor\n",
        "from lightgbm import LGBMRegressor\n"
      ],
      "metadata": {
        "id": "unqjH0Sn3imq"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Loading Dataset"
      ],
      "metadata": {
        "id": "hD3XN5aZ4X8c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(\"/content/insurance.csv\")\n"
      ],
      "metadata": {
        "id": "Y2kfIAfT1Frr"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Printing some dataset properties"
      ],
      "metadata": {
        "id": "HxiDD3GB4cfc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"*\"*30, \"HEAD\", \"*\"*30)\n",
        "display(data.head(5))\n",
        "print(\"*\"*30, \"SHAPE\", \"*\"*30)\n",
        "print(f\"Rows: {data.shape[0]}\\nColumns: {data.shape[1]}\")\n",
        "print(\"*\"*30, \"INFO\", \"*\"*30)\n",
        "display(data.info())\n",
        "print(\"*\"*30, \"DESCRIBE\", \"*\"*30)\n",
        "display(data.describe().T)\n",
        "print(\"*\"*30, \"NULL?\", \"*\"*30)\n",
        "display(data.isnull().sum())\n",
        "print(\"*\"*30, \"DUPLICATED\", \"*\"*30)\n",
        "display(data.duplicated().sum())\n",
        "print(\"*\"*30, \"EXPLAINING\", \"*\"*30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 937
        },
        "id": "T-D8Q12O1L1y",
        "outputId": "a84eaeca-a981-4453-81e0-7d7dc1d9d4ca"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************** HEAD ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "   age     sex     bmi  children smoker     region      charges\n",
              "0   19  female  27.900         0    yes  southwest  16884.92400\n",
              "1   18    male  33.770         1     no  southeast   1725.55230\n",
              "2   28    male  33.000         3     no  southeast   4449.46200\n",
              "3   33    male  22.705         0     no  northwest  21984.47061\n",
              "4   32    male  28.880         0     no  northwest   3866.85520"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6b2bd0bb-9a40-4015-b278-be23c941b273\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>age</th>\n",
              "      <th>sex</th>\n",
              "      <th>bmi</th>\n",
              "      <th>children</th>\n",
              "      <th>smoker</th>\n",
              "      <th>region</th>\n",
              "      <th>charges</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>19</td>\n",
              "      <td>female</td>\n",
              "      <td>27.900</td>\n",
              "      <td>0</td>\n",
              "      <td>yes</td>\n",
              "      <td>southwest</td>\n",
              "      <td>16884.92400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>18</td>\n",
              "      <td>male</td>\n",
              "      <td>33.770</td>\n",
              "      <td>1</td>\n",
              "      <td>no</td>\n",
              "      <td>southeast</td>\n",
              "      <td>1725.55230</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>28</td>\n",
              "      <td>male</td>\n",
              "      <td>33.000</td>\n",
              "      <td>3</td>\n",
              "      <td>no</td>\n",
              "      <td>southeast</td>\n",
              "      <td>4449.46200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>33</td>\n",
              "      <td>male</td>\n",
              "      <td>22.705</td>\n",
              "      <td>0</td>\n",
              "      <td>no</td>\n",
              "      <td>northwest</td>\n",
              "      <td>21984.47061</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>32</td>\n",
              "      <td>male</td>\n",
              "      <td>28.880</td>\n",
              "      <td>0</td>\n",
              "      <td>no</td>\n",
              "      <td>northwest</td>\n",
              "      <td>3866.85520</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6b2bd0bb-9a40-4015-b278-be23c941b273')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6b2bd0bb-9a40-4015-b278-be23c941b273 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6b2bd0bb-9a40-4015-b278-be23c941b273');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************** SHAPE ******************************\n",
            "Rows: 1338\n",
            "Columns: 7\n",
            "****************************** INFO ******************************\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1338 entries, 0 to 1337\n",
            "Data columns (total 7 columns):\n",
            " #   Column    Non-Null Count  Dtype  \n",
            "---  ------    --------------  -----  \n",
            " 0   age       1338 non-null   int64  \n",
            " 1   sex       1338 non-null   object \n",
            " 2   bmi       1338 non-null   float64\n",
            " 3   children  1338 non-null   int64  \n",
            " 4   smoker    1338 non-null   object \n",
            " 5   region    1338 non-null   object \n",
            " 6   charges   1338 non-null   float64\n",
            "dtypes: float64(2), int64(2), object(3)\n",
            "memory usage: 73.3+ KB\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "None"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************** DESCRIBE ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "           count          mean           std        min         25%       50%  \\\n",
              "age       1338.0     39.207025     14.049960    18.0000    27.00000    39.000   \n",
              "bmi       1338.0     30.663397      6.098187    15.9600    26.29625    30.400   \n",
              "children  1338.0      1.094918      1.205493     0.0000     0.00000     1.000   \n",
              "charges   1338.0  13270.422265  12110.011237  1121.8739  4740.28715  9382.033   \n",
              "\n",
              "                   75%          max  \n",
              "age          51.000000     64.00000  \n",
              "bmi          34.693750     53.13000  \n",
              "children      2.000000      5.00000  \n",
              "charges   16639.912515  63770.42801  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e43b9d66-6834-4448-aa16-885335017375\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>mean</th>\n",
              "      <th>std</th>\n",
              "      <th>min</th>\n",
              "      <th>25%</th>\n",
              "      <th>50%</th>\n",
              "      <th>75%</th>\n",
              "      <th>max</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>age</th>\n",
              "      <td>1338.0</td>\n",
              "      <td>39.207025</td>\n",
              "      <td>14.049960</td>\n",
              "      <td>18.0000</td>\n",
              "      <td>27.00000</td>\n",
              "      <td>39.000</td>\n",
              "      <td>51.000000</td>\n",
              "      <td>64.00000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>bmi</th>\n",
              "      <td>1338.0</td>\n",
              "      <td>30.663397</td>\n",
              "      <td>6.098187</td>\n",
              "      <td>15.9600</td>\n",
              "      <td>26.29625</td>\n",
              "      <td>30.400</td>\n",
              "      <td>34.693750</td>\n",
              "      <td>53.13000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>children</th>\n",
              "      <td>1338.0</td>\n",
              "      <td>1.094918</td>\n",
              "      <td>1.205493</td>\n",
              "      <td>0.0000</td>\n",
              "      <td>0.00000</td>\n",
              "      <td>1.000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>5.00000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>charges</th>\n",
              "      <td>1338.0</td>\n",
              "      <td>13270.422265</td>\n",
              "      <td>12110.011237</td>\n",
              "      <td>1121.8739</td>\n",
              "      <td>4740.28715</td>\n",
              "      <td>9382.033</td>\n",
              "      <td>16639.912515</td>\n",
              "      <td>63770.42801</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e43b9d66-6834-4448-aa16-885335017375')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e43b9d66-6834-4448-aa16-885335017375 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e43b9d66-6834-4448-aa16-885335017375');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************** NULL? ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "age         0\n",
              "sex         0\n",
              "bmi         0\n",
              "children    0\n",
              "smoker      0\n",
              "region      0\n",
              "charges     0\n",
              "dtype: int64"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************** DUPLICATED ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************** EXPLAINING ******************************\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##We have Rows: 1338, Columns: 7\n",
        "##There is no null values\n",
        "##Data have 1 duplicated values, we need to drop them."
      ],
      "metadata": {
        "id": "tveUcJ5Q2MJ_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data.drop_duplicates(inplace=True)"
      ],
      "metadata": {
        "id": "21Hl0yE42EWM"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Preparing Data\n",
        "###Feature Label Split "
      ],
      "metadata": {
        "id": "B2e3DN3W4zsU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Y = data[\"charges\"]\n",
        "X = data.drop(labels = [\"charges\"],axis=1)\n"
      ],
      "metadata": {
        "id": "HMkjlnC03ohy"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Encoding the Object columns to achieve full numerical dataset"
      ],
      "metadata": {
        "id": "oAFz2_jc4yM8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "le = LabelEncoder()\n",
        "X[\"sex\"]  = le.fit_transform(X[\"sex\"])\n",
        "X[\"smoker\"]  = le.fit_transform(X[\"smoker\"])\n",
        "X[\"region\"]  = le.fit_transform(X[\"region\"])"
      ],
      "metadata": {
        "id": "7jr1SDUG98tW"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "34jIK-P4-uXG",
        "outputId": "bb92c5c2-f5e1-4c4b-cc68-152efbec07b9"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   age  sex     bmi  children  smoker  region\n",
              "0   19    0  27.900         0       1       3\n",
              "1   18    1  33.770         1       0       2\n",
              "2   28    1  33.000         3       0       2\n",
              "3   33    1  22.705         0       0       1\n",
              "4   32    1  28.880         0       0       1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-50471933-318a-42b3-81c1-4ea8f7434915\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>age</th>\n",
              "      <th>sex</th>\n",
              "      <th>bmi</th>\n",
              "      <th>children</th>\n",
              "      <th>smoker</th>\n",
              "      <th>region</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>19</td>\n",
              "      <td>0</td>\n",
              "      <td>27.900</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>18</td>\n",
              "      <td>1</td>\n",
              "      <td>33.770</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>28</td>\n",
              "      <td>1</td>\n",
              "      <td>33.000</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>33</td>\n",
              "      <td>1</td>\n",
              "      <td>22.705</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>32</td>\n",
              "      <td>1</td>\n",
              "      <td>28.880</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-50471933-318a-42b3-81c1-4ea8f7434915')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-50471933-318a-42b3-81c1-4ea8f7434915 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-50471933-318a-42b3-81c1-4ea8f7434915');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Train Test Split"
      ],
      "metadata": {
        "id": "OTIBnXSE4096"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, random_state=42)"
      ],
      "metadata": {
        "id": "L1U1vzol5aKc"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Scaling the dataset"
      ],
      "metadata": {
        "id": "VwtKeJXD5G_e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "Xtrain = X_train.copy()\n",
        "Xtest = X_test.copy()\n",
        "\n",
        "#Standard Scaler\n",
        "scaler = StandardScaler()\n",
        "X_train_standard = scaler.fit_transform(X_train)\n",
        "X_test_standard = scaler.fit_transform(X_test)\n",
        "\n",
        "#Minmax Scaler\n",
        "scaler = MinMaxScaler()\n",
        "X_train_minmax = scaler.fit_transform(X_train)\n",
        "X_test_minmax = scaler.fit_transform(X_test)\n",
        "\n",
        "#Normalization\n",
        "X_train_normalize = preprocessing.normalize(X_train)\n",
        "X_test_normalize = preprocessing.normalize(X_test)\n",
        "\n",
        "train_list = [Xtrain,X_train_standard,X_train_minmax,X_train_normalize]\n",
        "scaler_list = [\"without_scaler\",\"standard_scaler\",\"minmax_scaler\",\"normalize\"]"
      ],
      "metadata": {
        "id": "BEtb0IL64mpM"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### We used two different scaler and one normalizer and raw data to try out all types and decide which gives the best score"
      ],
      "metadata": {
        "id": "RRBhiHNJ5NjM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#MODEL Selection\n"
      ],
      "metadata": {
        "id": "ZgdR8tMw-3Us"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### We used 15 different models and 4 differently scaled data and choosed the best performing couple according to R2 score"
      ],
      "metadata": {
        "id": "02zwHnSe5mE8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "kfold = StratifiedKFold(n_splits=10)\n",
        "random_state = 42\n",
        "z = 0\n",
        "\n",
        "for i in train_list:\n",
        "    linear_model = LinearRegression().fit(i,y_train)\n",
        "    knn_model = KNeighborsRegressor().fit(i, y_train)\n",
        "    decision_model = DecisionTreeRegressor().fit(i,y_train)\n",
        "    mlp_model = MLPRegressor().fit(i, y_train)\n",
        "    gaussian_model = BayesianRidge().fit(i, y_train)\n",
        "    linear_svm_model = SVR(kernel='linear').fit(i,y_train)\n",
        "    adaboost_model = AdaBoostRegressor(DecisionTreeRegressor(), learning_rate=0.1).fit(i,y_train)\n",
        "    randomforest_model = RandomForestRegressor().fit(i,y_train)\n",
        "    extra_model = ExtraTreesRegressor().fit(i,y_train)\n",
        "    gb_model = GradientBoostingRegressor().fit(i,y_train)\n",
        "    xgb_model = XGBRegressor().fit(i,y_train)\n",
        "    lgbm_model = LGBMRegressor().fit(i,y_train)\n",
        "    elastic_model = ElasticNet().fit(i,y_train)\n",
        "    sgd_model = SGDRegressor().fit(i,y_train)\n",
        "    kernel_model = KernelRidge().fit(i,y_train)\n",
        "    model_names = [\"Linear\",\"Knn\",\"DecisionTree\",\"MLP\",\"GaussianNB\",\"SupportVectorMachine\",\"AdaBoost\",\"RandomForest\",\"ExtraTrees\",\"GradientBoost\",\"Xgboost\",\"Lightgbm\", \"Elastic\", \"SGD\", \"Kernel Ridge\"]\n",
        "    model_list = [linear_model,knn_model,decision_model,mlp_model,gaussian_model,linear_svm_model,adaboost_model,randomforest_model,extra_model,gb_model,xgb_model,lgbm_model,elastic_model,sgd_model,kernel_model]\n",
        "    results = []\n",
        "    z +=1\n",
        "    if z ==1:\n",
        "        print(\"*\"*30, f\"{scaler_list[z-1]}\",\"*\"*30)\n",
        "    if z ==2:\n",
        "        print(\"*\"*30, f\"{scaler_list[z-1]}\",\"*\"*30)\n",
        "    if z ==3:\n",
        "        print(\"*\"*30, f\"{scaler_list[z-1]}\",\"*\"*30)\n",
        "    if z ==4:\n",
        "        print(\"*\"*30, f\"{scaler_list[z-1]}\",\"*\"*30)\n",
        "    for j in model_list:\n",
        "        result = cross_val_score(j, i, y_train, scoring = \"r2\", cv = 5, n_jobs=4)\n",
        "        results.append(result.mean())\n",
        "\n",
        "    acc_of_models = {\"Model\": model_names, \"R2\": results}    \n",
        "    acc_of_models = pd.DataFrame(acc_of_models)\n",
        "    display(acc_of_models)\n",
        "    print(np.max(acc_of_models[\"R2\"]))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "jTOcWJaD-1jV",
        "outputId": "67e1991c-a197-48a1-82b3-4b9de006f4e0"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[13:23:03] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "****************************** without_scaler ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   Model            R2\n",
              "0                 Linear  7.292348e-01\n",
              "1                    Knn  3.195326e-02\n",
              "2           DecisionTree  6.688155e-01\n",
              "3                    MLP -4.505553e-01\n",
              "4             GaussianNB  7.292580e-01\n",
              "5   SupportVectorMachine -1.608480e-01\n",
              "6               AdaBoost  8.004527e-01\n",
              "7           RandomForest  8.247454e-01\n",
              "8             ExtraTrees  8.034135e-01\n",
              "9          GradientBoost  8.417020e-01\n",
              "10               Xgboost  8.456784e-01\n",
              "11              Lightgbm  8.229682e-01\n",
              "12               Elastic  3.713162e-01\n",
              "13                   SGD -1.780624e+14\n",
              "14          Kernel Ridge  7.007588e-01"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-409f469d-7ceb-4643-a449-45ce201067f2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Model</th>\n",
              "      <th>R2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Linear</td>\n",
              "      <td>7.292348e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Knn</td>\n",
              "      <td>3.195326e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DecisionTree</td>\n",
              "      <td>6.688155e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MLP</td>\n",
              "      <td>-4.505553e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GaussianNB</td>\n",
              "      <td>7.292580e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SupportVectorMachine</td>\n",
              "      <td>-1.608480e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AdaBoost</td>\n",
              "      <td>8.004527e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>RandomForest</td>\n",
              "      <td>8.247454e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>8.034135e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>GradientBoost</td>\n",
              "      <td>8.417020e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Xgboost</td>\n",
              "      <td>8.456784e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Lightgbm</td>\n",
              "      <td>8.229682e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Elastic</td>\n",
              "      <td>3.713162e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>SGD</td>\n",
              "      <td>-1.780624e+14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Kernel Ridge</td>\n",
              "      <td>7.007588e-01</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-409f469d-7ceb-4643-a449-45ce201067f2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-409f469d-7ceb-4643-a449-45ce201067f2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-409f469d-7ceb-4643-a449-45ce201067f2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8456784125717617\n",
            "[13:23:22] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "****************************** standard_scaler ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   Model        R2\n",
              "0                 Linear  0.729235\n",
              "1                    Knn  0.786199\n",
              "2           DecisionTree  0.674978\n",
              "3                    MLP -1.212540\n",
              "4             GaussianNB  0.729264\n",
              "5   SupportVectorMachine -0.040895\n",
              "6               AdaBoost  0.803707\n",
              "7           RandomForest  0.823525\n",
              "8             ExtraTrees  0.799969\n",
              "9          GradientBoost  0.841667\n",
              "10               Xgboost  0.845680\n",
              "11              Lightgbm  0.825720\n",
              "12               Elastic  0.647581\n",
              "13                   SGD  0.729253\n",
              "14          Kernel Ridge -0.551853"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-52dd6c41-58e4-434d-9db3-326c1e286f23\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Model</th>\n",
              "      <th>R2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Linear</td>\n",
              "      <td>0.729235</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Knn</td>\n",
              "      <td>0.786199</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DecisionTree</td>\n",
              "      <td>0.674978</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MLP</td>\n",
              "      <td>-1.212540</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GaussianNB</td>\n",
              "      <td>0.729264</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SupportVectorMachine</td>\n",
              "      <td>-0.040895</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AdaBoost</td>\n",
              "      <td>0.803707</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>RandomForest</td>\n",
              "      <td>0.823525</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.799969</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>GradientBoost</td>\n",
              "      <td>0.841667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Xgboost</td>\n",
              "      <td>0.845680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Lightgbm</td>\n",
              "      <td>0.825720</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Elastic</td>\n",
              "      <td>0.647581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>SGD</td>\n",
              "      <td>0.729253</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Kernel Ridge</td>\n",
              "      <td>-0.551853</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-52dd6c41-58e4-434d-9db3-326c1e286f23')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-52dd6c41-58e4-434d-9db3-326c1e286f23 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-52dd6c41-58e4-434d-9db3-326c1e286f23');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8456798152695658\n",
            "[13:23:38] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "****************************** minmax_scaler ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   Model        R2\n",
              "0                 Linear  0.729235\n",
              "1                    Knn  0.735039\n",
              "2           DecisionTree  0.662807\n",
              "3                    MLP -1.192378\n",
              "4             GaussianNB  0.729259\n",
              "5   SupportVectorMachine -0.096059\n",
              "6               AdaBoost  0.799921\n",
              "7           RandomForest  0.820188\n",
              "8             ExtraTrees  0.804231\n",
              "9          GradientBoost  0.841598\n",
              "10               Xgboost  0.845703\n",
              "11              Lightgbm  0.825869\n",
              "12               Elastic  0.288148\n",
              "13                   SGD  0.728206\n",
              "14          Kernel Ridge  0.727408"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d7fb294c-68be-422b-9eac-9d9289fe597c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Model</th>\n",
              "      <th>R2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Linear</td>\n",
              "      <td>0.729235</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Knn</td>\n",
              "      <td>0.735039</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DecisionTree</td>\n",
              "      <td>0.662807</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MLP</td>\n",
              "      <td>-1.192378</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GaussianNB</td>\n",
              "      <td>0.729259</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SupportVectorMachine</td>\n",
              "      <td>-0.096059</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AdaBoost</td>\n",
              "      <td>0.799921</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>RandomForest</td>\n",
              "      <td>0.820188</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.804231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>GradientBoost</td>\n",
              "      <td>0.841598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Xgboost</td>\n",
              "      <td>0.845703</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Lightgbm</td>\n",
              "      <td>0.825869</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Elastic</td>\n",
              "      <td>0.288148</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>SGD</td>\n",
              "      <td>0.728206</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Kernel Ridge</td>\n",
              "      <td>0.727408</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d7fb294c-68be-422b-9eac-9d9289fe597c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d7fb294c-68be-422b-9eac-9d9289fe597c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d7fb294c-68be-422b-9eac-9d9289fe597c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8457029580999172\n",
            "[13:23:45] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "****************************** normalize ******************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   Model        R2\n",
              "0                 Linear  0.523495\n",
              "1                    Knn  0.215502\n",
              "2           DecisionTree  0.663077\n",
              "3                    MLP -1.232832\n",
              "4             GaussianNB  0.523561\n",
              "5   SupportVectorMachine -0.106283\n",
              "6               AdaBoost  0.787570\n",
              "7           RandomForest  0.812685\n",
              "8             ExtraTrees  0.794174\n",
              "9          GradientBoost  0.803789\n",
              "10               Xgboost  0.807086\n",
              "11              Lightgbm  0.789530\n",
              "12               Elastic -0.000663\n",
              "13                   SGD  0.026251\n",
              "14          Kernel Ridge  0.078244"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6fe36d78-2726-4987-a54c-b612959bd7ab\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Model</th>\n",
              "      <th>R2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Linear</td>\n",
              "      <td>0.523495</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Knn</td>\n",
              "      <td>0.215502</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>DecisionTree</td>\n",
              "      <td>0.663077</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>MLP</td>\n",
              "      <td>-1.232832</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>GaussianNB</td>\n",
              "      <td>0.523561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SupportVectorMachine</td>\n",
              "      <td>-0.106283</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AdaBoost</td>\n",
              "      <td>0.787570</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>RandomForest</td>\n",
              "      <td>0.812685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ExtraTrees</td>\n",
              "      <td>0.794174</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>GradientBoost</td>\n",
              "      <td>0.803789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Xgboost</td>\n",
              "      <td>0.807086</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Lightgbm</td>\n",
              "      <td>0.789530</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Elastic</td>\n",
              "      <td>-0.000663</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>SGD</td>\n",
              "      <td>0.026251</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Kernel Ridge</td>\n",
              "      <td>0.078244</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6fe36d78-2726-4987-a54c-b612959bd7ab')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6fe36d78-2726-4987-a54c-b612959bd7ab button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6fe36d78-2726-4987-a54c-b612959bd7ab');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8126848999763527\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#The best score was XGBRegrossor algorithm trained with min-max scaled data"
      ],
      "metadata": {
        "id": "AtrgY6F852Pl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = XGBRegressor().fit(X_train_minmax,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UEt9GyX6lndU",
        "outputId": "c26ea463-faac-4763-f52c-ad75fc94b7c5"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[13:23:52] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Model *Train* Score"
      ],
      "metadata": {
        "id": "CUaRQS_r6Abt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.score(X_train_minmax,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OChV2k9jnAjd",
        "outputId": "5281bf2c-d141-4b5f-cf14-f5387636e867"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.897703181488283"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Model *Test* Score"
      ],
      "metadata": {
        "id": "fX8V2yhO6JfE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "y_preds = model.predict(X_test_minmax)\n",
        "from sklearn.metrics import r2_score\n",
        "\n",
        "r2_score(y_test, y_preds)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ly0L2toeo-hd",
        "outputId": "d96167ad-f4a6-4591-c558-a5f0caf0bcd3"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8643217022421513"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Hyperparameter Tuning"
      ],
      "metadata": {
        "id": "_4tZY4nr6Mrc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def algorithm_pipeline(X_train_data, X_test_data, y_train_data, y_test_data, \n",
        "                       model, param_grid, cv=10, scoring_fit='r2',\n",
        "                       do_probabilities = False):\n",
        "    gs = GridSearchCV(\n",
        "        estimator=model,\n",
        "        param_grid=param_grid, \n",
        "        cv=cv, \n",
        "        n_jobs=-1, \n",
        "        scoring=scoring_fit,\n",
        "        verbose=2\n",
        "    )\n",
        "    fitted_model = gs.fit(X_train_data, y_train_data)\n",
        "    \n",
        "    if do_probabilities:\n",
        "      pred = fitted_model.predict_proba(X_test_data)\n",
        "    else:\n",
        "      pred = fitted_model.predict(X_test_data)\n",
        "    \n",
        "    return fitted_model, pred"
      ],
      "metadata": {
        "id": "Yncn57h8pVYs"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = XGBRegressor()\n",
        "param_grid = {\n",
        "    'n_estimators': [200, 300, 400],\n",
        "    'colsample_bytree': [0.7, 0.8],\n",
        "    'max_depth': [15,20,25],\n",
        "    'reg_alpha': [1.1, 1.2, 1.3],\n",
        "    'reg_lambda': [1.1, 1.2, 1.3],\n",
        "    'subsample': [0.7, 0.8, 0.9]\n",
        "}\n",
        "\n",
        "model, pred = algorithm_pipeline(X_train_minmax, X_test_minmax, y_train, y_test, model, \n",
        "                                 param_grid, cv=5)\n",
        "\n",
        "# Root Mean Squared Error\n",
        "print(model.best_score_)\n",
        "print(model.best_params_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nIfj4NQ4pXFS",
        "outputId": "f312a40f-313a-4fff-ff85-a36f797c0962"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 486 candidates, totalling 2430 fits\n",
            "[13:44:37] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "0.7688396759242134\n",
            "{'colsample_bytree': 0.7, 'max_depth': 15, 'n_estimators': 200, 'reg_alpha': 1.2, 'reg_lambda': 1.3, 'subsample': 0.7}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(r2_score(y_test, pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kFpY2sin67ht",
        "outputId": "7c8ef137-1554-40e8-f587-dcb10fccbb2f"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.829471706765329\n"
          ]
        }
      ]
    }
  ]
}