BaysianHyperparameter_optimization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BaysianHyperparameter optimization.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ravi02512/AM-Expert-Analytics-Vidhya-Hackathon/blob/master/BaysianHyperparameter_optimization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qROv1Mc1nJBj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "outputId": "b4b9565f-53c4-486e-a539-c4e1744d7c8b"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1IDO2G-InZLP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "train=pd.read_csv('/content/gdrive/My Drive/null_False_final.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lOOdyH6vnuA5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_df = train[train['redemption_status'].notnull()]\n",
        "test_df = train[train['redemption_status'].isnull()]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mVLIC3qGnZTT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X=train_df.drop('redemption_status', axis=1)\n",
        "y=train_df['redemption_status']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9IrjWuaHnZXg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "7f204881-06e7-4991-a427-1ce282f33b78"
      },
      "source": [
        "!pip install bayesian-optimization"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting bayesian-optimization\n",
            "  Downloading https://files.pythonhosted.org/packages/72/0c/173ac467d0a53e33e41b521e4ceba74a8ac7c7873d7b857a8fbdca88302d/bayesian-optimization-1.0.1.tar.gz\n",
            "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from bayesian-optimization) (1.16.5)\n",
            "Requirement already satisfied: scipy>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from bayesian-optimization) (1.3.1)\n",
            "Requirement already satisfied: scikit-learn>=0.18.0 in /usr/local/lib/python3.6/dist-packages (from bayesian-optimization) (0.21.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18.0->bayesian-optimization) (0.13.2)\n",
            "Building wheels for collected packages: bayesian-optimization\n",
            "  Building wheel for bayesian-optimization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for bayesian-optimization: filename=bayesian_optimization-1.0.1-cp36-none-any.whl size=10032 sha256=0164d809c84c98ee12a3e1f8d82b8402f8701fe79abc9cb0d04ce7b784454207\n",
            "  Stored in directory: /root/.cache/pip/wheels/1d/0d/3b/6b9d4477a34b3905f246ff4e7acf6aafd4cc9b77d473629b77\n",
            "Successfully built bayesian-optimization\n",
            "Installing collected packages: bayesian-optimization\n",
            "Successfully installed bayesian-optimization-1.0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gIPo7DzhnZa9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### Hyperparameter optimization \n",
        "import warnings\n",
        "import time\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import lightgbm as lgb\n",
        "from bayes_opt import BayesianOptimization\n",
        "from sklearn.metrics import roc_auc_score\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gq1CGUCcnZeM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "outputId": "b81a8bdd-7d5d-4c91-8db8-8c1ff3aa6f5b"
      },
      "source": [
        "def bayes_parameter_opt_lgb(X, y, init_round=15, opt_round=25, n_folds=5, random_seed=6, n_estimators=10000, learning_rate=0.05, output_process=False):\n",
        "    # prepare data\n",
        "    train_data = lgb.Dataset(data=X, label=y, free_raw_data=False)\n",
        "    # parameters\n",
        "    def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight):\n",
        "        params = {'application':'binary','num_iterations': n_estimators, 'learning_rate':learning_rate, 'early_stopping_round':100, 'metric':'auc'}\n",
        "        params[\"num_leaves\"] = int(round(num_leaves))\n",
        "        params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n",
        "        params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n",
        "        params['max_depth'] = int(round(max_depth))\n",
        "        params['lambda_l1'] = max(lambda_l1, 0)\n",
        "        params['lambda_l2'] = max(lambda_l2, 0)\n",
        "        params['min_split_gain'] = min_split_gain\n",
        "        params['min_child_weight'] = min_child_weight\n",
        "        cv_result = lgb.cv(params, train_data, nfold=n_folds, seed=random_seed, stratified=True, verbose_eval =200, metrics=['auc'])\n",
        "        return max(cv_result['auc-mean'])\n",
        "    # range \n",
        "    lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (24, 45),\n",
        "                                            'feature_fraction': (0.1, 0.9),\n",
        "                                            'bagging_fraction': (0.8, 1),\n",
        "                                            'max_depth': (5, 8.99),\n",
        "                                            'lambda_l1': (0, 5),\n",
        "                                            'lambda_l2': (0, 3),\n",
        "                                            'min_split_gain': (0.001, 0.1),\n",
        "                                            'min_child_weight': (5, 50)}, random_state=0)\n",
        "    # optimize\n",
        "    lgbBO.maximize(init_points=init_round, n_iter=opt_round)\n",
        "    \n",
        "    # output optimization process\n",
        "    if output_process==True: \n",
        "      lgbBO.points_to_csv(\"bayes_opt_result.csv\")\n",
        "    \n",
        "    # return best parameters\n",
        "    return lgbBO.max\n",
        "\n",
        "opt_params = bayes_parameter_opt_lgb(X, y, init_round=5, opt_round=20, n_folds=5, random_seed=6, n_estimators=100, learning_rate=0.005)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "|   iter    |  target   | baggin... | featur... | lambda_l1 | lambda_l2 | max_depth | min_ch... | min_sp... | num_le... |\n",
            "-------------------------------------------------------------------------------------------------------------------------\n",
            "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.8819  \u001b[0m | \u001b[0m 0.9098  \u001b[0m | \u001b[0m 0.6722  \u001b[0m | \u001b[0m 3.014   \u001b[0m | \u001b[0m 1.635   \u001b[0m | \u001b[0m 6.69    \u001b[0m | \u001b[0m 34.07   \u001b[0m | \u001b[0m 0.04432 \u001b[0m | \u001b[0m 42.73   \u001b[0m |\n",
            "| \u001b[0m 2       \u001b[0m | \u001b[0m 0.8769  \u001b[0m | \u001b[0m 0.9927  \u001b[0m | \u001b[0m 0.4068  \u001b[0m | \u001b[0m 3.959   \u001b[0m | \u001b[0m 1.587   \u001b[0m | \u001b[0m 7.266   \u001b[0m | \u001b[0m 46.65   \u001b[0m | \u001b[0m 0.008033\u001b[0m | \u001b[0m 25.83   \u001b[0m |\n",
            "| \u001b[0m 3       \u001b[0m | \u001b[0m 0.8792  \u001b[0m | \u001b[0m 0.804   \u001b[0m | \u001b[0m 0.7661  \u001b[0m | \u001b[0m 3.891   \u001b[0m | \u001b[0m 2.61    \u001b[0m | \u001b[0m 8.905   \u001b[0m | \u001b[0m 40.96   \u001b[0m | \u001b[0m 0.04669 \u001b[0m | \u001b[0m 40.39   \u001b[0m |\n",
            "| \u001b[95m 4       \u001b[0m | \u001b[95m 0.8907  \u001b[0m | \u001b[95m 0.8237  \u001b[0m | \u001b[95m 0.6119  \u001b[0m | \u001b[95m 0.7168  \u001b[0m | \u001b[95m 2.834   \u001b[0m | \u001b[95m 7.082   \u001b[0m | \u001b[95m 23.66   \u001b[0m | \u001b[95m 0.02719 \u001b[0m | \u001b[95m 40.26   \u001b[0m |\n",
            "| \u001b[0m 5       \u001b[0m | \u001b[0m 0.8849  \u001b[0m | \u001b[0m 0.8912  \u001b[0m | \u001b[0m 0.5547  \u001b[0m | \u001b[0m 0.09395 \u001b[0m | \u001b[0m 1.853   \u001b[0m | \u001b[0m 7.442   \u001b[0m | \u001b[0m 32.76   \u001b[0m | \u001b[0m 0.09443 \u001b[0m | \u001b[0m 38.32   \u001b[0m |\n",
            "| \u001b[95m 6       \u001b[0m | \u001b[95m 0.8968  \u001b[0m | \u001b[95m 0.8869  \u001b[0m | \u001b[95m 0.2392  \u001b[0m | \u001b[95m 4.977   \u001b[0m | \u001b[95m 2.348   \u001b[0m | \u001b[95m 8.828   \u001b[0m | \u001b[95m 5.161   \u001b[0m | \u001b[95m 0.00634 \u001b[0m | \u001b[95m 24.55   \u001b[0m |\n",
            "| \u001b[95m 7       \u001b[0m | \u001b[95m 0.9115  \u001b[0m | \u001b[95m 0.8598  \u001b[0m | \u001b[95m 0.8069  \u001b[0m | \u001b[95m 0.1164  \u001b[0m | \u001b[95m 0.3642  \u001b[0m | \u001b[95m 5.009   \u001b[0m | \u001b[95m 5.017   \u001b[0m | \u001b[95m 0.07182 \u001b[0m | \u001b[95m 24.98   \u001b[0m |\n",
            "| \u001b[95m 8       \u001b[0m | \u001b[95m 0.9144  \u001b[0m | \u001b[95m 0.9478  \u001b[0m | \u001b[95m 0.8198  \u001b[0m | \u001b[95m 0.8939  \u001b[0m | \u001b[95m 0.2559  \u001b[0m | \u001b[95m 8.792   \u001b[0m | \u001b[95m 5.524   \u001b[0m | \u001b[95m 0.002175\u001b[0m | \u001b[95m 44.6    \u001b[0m |\n",
            "| \u001b[0m 9       \u001b[0m | \u001b[0m 0.9053  \u001b[0m | \u001b[0m 0.8228  \u001b[0m | \u001b[0m 0.3484  \u001b[0m | \u001b[0m 0.1293  \u001b[0m | \u001b[0m 2.851   \u001b[0m | \u001b[0m 6.086   \u001b[0m | \u001b[0m 5.028   \u001b[0m | \u001b[0m 0.03605 \u001b[0m | \u001b[0m 43.45   \u001b[0m |\n",
            "| \u001b[0m 10      \u001b[0m | \u001b[0m 0.9114  \u001b[0m | \u001b[0m 0.8354  \u001b[0m | \u001b[0m 0.4607  \u001b[0m | \u001b[0m 0.1239  \u001b[0m | \u001b[0m 0.2041  \u001b[0m | \u001b[0m 8.83    \u001b[0m | \u001b[0m 5.242   \u001b[0m | \u001b[0m 0.02661 \u001b[0m | \u001b[0m 39.88   \u001b[0m |\n",
            "| \u001b[95m 11      \u001b[0m | \u001b[95m 0.9168  \u001b[0m | \u001b[95m 0.8014  \u001b[0m | \u001b[95m 0.8874  \u001b[0m | \u001b[95m 0.000690\u001b[0m | \u001b[95m 0.009321\u001b[0m | \u001b[95m 7.924   \u001b[0m | \u001b[95m 5.095   \u001b[0m | \u001b[95m 0.03327 \u001b[0m | \u001b[95m 42.82   \u001b[0m |\n",
            "| \u001b[0m 12      \u001b[0m | \u001b[0m 0.9165  \u001b[0m | \u001b[0m 0.9946  \u001b[0m | \u001b[0m 0.8065  \u001b[0m | \u001b[0m 0.2236  \u001b[0m | \u001b[0m 0.1132  \u001b[0m | \u001b[0m 8.45    \u001b[0m | \u001b[0m 5.069   \u001b[0m | \u001b[0m 0.08541 \u001b[0m | \u001b[0m 43.86   \u001b[0m |\n",
            "| \u001b[0m 13      \u001b[0m | \u001b[0m 0.9147  \u001b[0m | \u001b[0m 0.8051  \u001b[0m | \u001b[0m 0.8919  \u001b[0m | \u001b[0m 1.397   \u001b[0m | \u001b[0m 0.01415 \u001b[0m | \u001b[0m 8.78    \u001b[0m | \u001b[0m 5.157   \u001b[0m | \u001b[0m 0.04927 \u001b[0m | \u001b[0m 42.77   \u001b[0m |\n",
            "| \u001b[0m 14      \u001b[0m | \u001b[0m 0.9084  \u001b[0m | \u001b[0m 0.873   \u001b[0m | \u001b[0m 0.8392  \u001b[0m | \u001b[0m 1.536   \u001b[0m | \u001b[0m 0.4962  \u001b[0m | \u001b[0m 5.018   \u001b[0m | \u001b[0m 5.251   \u001b[0m | \u001b[0m 0.04583 \u001b[0m | \u001b[0m 44.91   \u001b[0m |\n",
            "| \u001b[0m 15      \u001b[0m | \u001b[0m 0.9159  \u001b[0m | \u001b[0m 0.8199  \u001b[0m | \u001b[0m 0.841   \u001b[0m | \u001b[0m 0.05802 \u001b[0m | \u001b[0m 0.1896  \u001b[0m | \u001b[0m 8.306   \u001b[0m | \u001b[0m 5.004   \u001b[0m | \u001b[0m 0.03067 \u001b[0m | \u001b[0m 33.43   \u001b[0m |\n",
            "| \u001b[0m 16      \u001b[0m | \u001b[0m 0.9142  \u001b[0m | \u001b[0m 0.8926  \u001b[0m | \u001b[0m 0.832   \u001b[0m | \u001b[0m 0.1746  \u001b[0m | \u001b[0m 0.1954  \u001b[0m | \u001b[0m 8.903   \u001b[0m | \u001b[0m 5.3     \u001b[0m | \u001b[0m 0.02069 \u001b[0m | \u001b[0m 28.64   \u001b[0m |\n",
            "| \u001b[0m 17      \u001b[0m | \u001b[0m 0.9123  \u001b[0m | \u001b[0m 0.8097  \u001b[0m | \u001b[0m 0.8952  \u001b[0m | \u001b[0m 0.6954  \u001b[0m | \u001b[0m 0.004936\u001b[0m | \u001b[0m 8.929   \u001b[0m | \u001b[0m 5.631   \u001b[0m | \u001b[0m 0.03488 \u001b[0m | \u001b[0m 24.73   \u001b[0m |\n",
            "| \u001b[0m 18      \u001b[0m | \u001b[0m 0.9156  \u001b[0m | \u001b[0m 0.9312  \u001b[0m | \u001b[0m 0.8618  \u001b[0m | \u001b[0m 0.1113  \u001b[0m | \u001b[0m 0.296   \u001b[0m | \u001b[0m 8.794   \u001b[0m | \u001b[0m 6.329   \u001b[0m | \u001b[0m 0.0245  \u001b[0m | \u001b[0m 42.76   \u001b[0m |\n",
            "| \u001b[0m 19      \u001b[0m | \u001b[0m 0.9161  \u001b[0m | \u001b[0m 0.9195  \u001b[0m | \u001b[0m 0.8487  \u001b[0m | \u001b[0m 0.1229  \u001b[0m | \u001b[0m 0.3261  \u001b[0m | \u001b[0m 8.78    \u001b[0m | \u001b[0m 5.141   \u001b[0m | \u001b[0m 0.04948 \u001b[0m | \u001b[0m 36.66   \u001b[0m |\n",
            "| \u001b[0m 20      \u001b[0m | \u001b[0m 0.9125  \u001b[0m | \u001b[0m 0.8091  \u001b[0m | \u001b[0m 0.8822  \u001b[0m | \u001b[0m 0.1238  \u001b[0m | \u001b[0m 2.235   \u001b[0m | \u001b[0m 8.562   \u001b[0m | \u001b[0m 5.381   \u001b[0m | \u001b[0m 0.02322 \u001b[0m | \u001b[0m 40.62   \u001b[0m |\n",
            "| \u001b[0m 21      \u001b[0m | \u001b[0m 0.9155  \u001b[0m | \u001b[0m 0.9436  \u001b[0m | \u001b[0m 0.8693  \u001b[0m | \u001b[0m 0.2785  \u001b[0m | \u001b[0m 0.4213  \u001b[0m | \u001b[0m 8.538   \u001b[0m | \u001b[0m 5.297   \u001b[0m | \u001b[0m 0.07036 \u001b[0m | \u001b[0m 35.19   \u001b[0m |\n",
            "| \u001b[0m 22      \u001b[0m | \u001b[0m 0.9157  \u001b[0m | \u001b[0m 0.9451  \u001b[0m | \u001b[0m 0.8118  \u001b[0m | \u001b[0m 0.3099  \u001b[0m | \u001b[0m 0.1804  \u001b[0m | \u001b[0m 8.109   \u001b[0m | \u001b[0m 5.126   \u001b[0m | \u001b[0m 0.03014 \u001b[0m | \u001b[0m 32.83   \u001b[0m |\n",
            "| \u001b[0m 23      \u001b[0m | \u001b[0m 0.915   \u001b[0m | \u001b[0m 0.9812  \u001b[0m | \u001b[0m 0.8952  \u001b[0m | \u001b[0m 0.7094  \u001b[0m | \u001b[0m 0.2703  \u001b[0m | \u001b[0m 8.683   \u001b[0m | \u001b[0m 5.231   \u001b[0m | \u001b[0m 0.05581 \u001b[0m | \u001b[0m 32.93   \u001b[0m |\n",
            "| \u001b[0m 24      \u001b[0m | \u001b[0m 0.9157  \u001b[0m | \u001b[0m 0.9932  \u001b[0m | \u001b[0m 0.7914  \u001b[0m | \u001b[0m 0.1605  \u001b[0m | \u001b[0m 0.03796 \u001b[0m | \u001b[0m 8.472   \u001b[0m | \u001b[0m 5.079   \u001b[0m | \u001b[0m 0.03447 \u001b[0m | \u001b[0m 36.39   \u001b[0m |\n",
            "| \u001b[0m 25      \u001b[0m | \u001b[0m 0.9166  \u001b[0m | \u001b[0m 0.9242  \u001b[0m | \u001b[0m 0.815   \u001b[0m | \u001b[0m 0.1467  \u001b[0m | \u001b[0m 0.2825  \u001b[0m | \u001b[0m 8.234   \u001b[0m | \u001b[0m 5.184   \u001b[0m | \u001b[0m 0.09322 \u001b[0m | \u001b[0m 44.84   \u001b[0m |\n",
            "=========================================================================================================================\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}