58181_catboost&skl_cross_validate.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "58181-catboost&skl_cross_validate.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bmreiniger/datascience.stackexchange/blob/master/58181_catboost%26skl_cross_validate.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Y4oGHvPXMSJ",
        "colab_type": "code",
        "outputId": "41b62d58-d85c-435c-b6e4-23eecebcef43",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        }
      },
      "source": [
        "!pip install catboost"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting catboost\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ef/e9/41060f73ca5dcf604f75bc871ee5ce8dcb201897640b37c95aa8b1e139c8/catboost-0.16.5-cp36-none-manylinux1_x86_64.whl (61.9MB)\n",
            "\u001b[K     |████████████████████████████████| 61.9MB 642kB/s \n",
            "\u001b[?25hRequirement already satisfied: graphviz in /usr/local/lib/python3.6/dist-packages (from catboost) (0.10.1)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.6/dist-packages (from catboost) (3.6.1)\n",
            "Requirement already satisfied: pandas>=0.19.1 in /usr/local/lib/python3.6/dist-packages (from catboost) (0.24.2)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from catboost) (1.16.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from catboost) (1.12.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from catboost) (3.0.3)\n",
            "Requirement already satisfied: decorator>=4.0.6 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (4.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (2.21.0)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (1.3.3)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (2018.9)\n",
            "Requirement already satisfied: nbformat>=4.2 in /usr/local/lib/python3.6/dist-packages (from plotly->catboost) (4.4.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.19.1->catboost) (2.5.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (1.1.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->catboost) (2.4.2)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->plotly->catboost) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->plotly->catboost) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->plotly->catboost) (2019.6.16)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->plotly->catboost) (1.24.3)\n",
            "Requirement already satisfied: traitlets>=4.1 in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2->plotly->catboost) (4.3.2)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2->plotly->catboost) (2.6.0)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2->plotly->catboost) (0.2.0)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2->plotly->catboost) (4.5.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib->catboost) (41.2.0)\n",
            "Installing collected packages: catboost\n",
            "Successfully installed catboost-0.16.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YXQE4NtyaIUd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import cross_validate\n",
        "import catboost as cb\n",
        "from sklearn.datasets import load_breast_cancer\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H17A0H2mavtk",
        "colab_type": "code",
        "outputId": "d064069b-ec52-46d1-86eb-81603afc94fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "source": [
        "X, y = load_breast_cancer(return_X_y=True)\n",
        "X = pd.DataFrame(X)\n",
        "X.columns = ['col'+str(i) for i in range(30)]\n",
        "X['col0'] = X['col0'].apply(lambda x: str(int(x))+'a')\n",
        "X.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>col0</th>\n",
              "      <th>col1</th>\n",
              "      <th>col2</th>\n",
              "      <th>col3</th>\n",
              "      <th>col4</th>\n",
              "      <th>col5</th>\n",
              "      <th>col6</th>\n",
              "      <th>col7</th>\n",
              "      <th>col8</th>\n",
              "      <th>col9</th>\n",
              "      <th>col10</th>\n",
              "      <th>col11</th>\n",
              "      <th>col12</th>\n",
              "      <th>col13</th>\n",
              "      <th>col14</th>\n",
              "      <th>col15</th>\n",
              "      <th>col16</th>\n",
              "      <th>col17</th>\n",
              "      <th>col18</th>\n",
              "      <th>col19</th>\n",
              "      <th>col20</th>\n",
              "      <th>col21</th>\n",
              "      <th>col22</th>\n",
              "      <th>col23</th>\n",
              "      <th>col24</th>\n",
              "      <th>col25</th>\n",
              "      <th>col26</th>\n",
              "      <th>col27</th>\n",
              "      <th>col28</th>\n",
              "      <th>col29</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>17a</td>\n",
              "      <td>10.38</td>\n",
              "      <td>122.80</td>\n",
              "      <td>1001.0</td>\n",
              "      <td>0.11840</td>\n",
              "      <td>0.27760</td>\n",
              "      <td>0.3001</td>\n",
              "      <td>0.14710</td>\n",
              "      <td>0.2419</td>\n",
              "      <td>0.07871</td>\n",
              "      <td>1.0950</td>\n",
              "      <td>0.9053</td>\n",
              "      <td>8.589</td>\n",
              "      <td>153.40</td>\n",
              "      <td>0.006399</td>\n",
              "      <td>0.04904</td>\n",
              "      <td>0.05373</td>\n",
              "      <td>0.01587</td>\n",
              "      <td>0.03003</td>\n",
              "      <td>0.006193</td>\n",
              "      <td>25.38</td>\n",
              "      <td>17.33</td>\n",
              "      <td>184.60</td>\n",
              "      <td>2019.0</td>\n",
              "      <td>0.1622</td>\n",
              "      <td>0.6656</td>\n",
              "      <td>0.7119</td>\n",
              "      <td>0.2654</td>\n",
              "      <td>0.4601</td>\n",
              "      <td>0.11890</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>20a</td>\n",
              "      <td>17.77</td>\n",
              "      <td>132.90</td>\n",
              "      <td>1326.0</td>\n",
              "      <td>0.08474</td>\n",
              "      <td>0.07864</td>\n",
              "      <td>0.0869</td>\n",
              "      <td>0.07017</td>\n",
              "      <td>0.1812</td>\n",
              "      <td>0.05667</td>\n",
              "      <td>0.5435</td>\n",
              "      <td>0.7339</td>\n",
              "      <td>3.398</td>\n",
              "      <td>74.08</td>\n",
              "      <td>0.005225</td>\n",
              "      <td>0.01308</td>\n",
              "      <td>0.01860</td>\n",
              "      <td>0.01340</td>\n",
              "      <td>0.01389</td>\n",
              "      <td>0.003532</td>\n",
              "      <td>24.99</td>\n",
              "      <td>23.41</td>\n",
              "      <td>158.80</td>\n",
              "      <td>1956.0</td>\n",
              "      <td>0.1238</td>\n",
              "      <td>0.1866</td>\n",
              "      <td>0.2416</td>\n",
              "      <td>0.1860</td>\n",
              "      <td>0.2750</td>\n",
              "      <td>0.08902</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>19a</td>\n",
              "      <td>21.25</td>\n",
              "      <td>130.00</td>\n",
              "      <td>1203.0</td>\n",
              "      <td>0.10960</td>\n",
              "      <td>0.15990</td>\n",
              "      <td>0.1974</td>\n",
              "      <td>0.12790</td>\n",
              "      <td>0.2069</td>\n",
              "      <td>0.05999</td>\n",
              "      <td>0.7456</td>\n",
              "      <td>0.7869</td>\n",
              "      <td>4.585</td>\n",
              "      <td>94.03</td>\n",
              "      <td>0.006150</td>\n",
              "      <td>0.04006</td>\n",
              "      <td>0.03832</td>\n",
              "      <td>0.02058</td>\n",
              "      <td>0.02250</td>\n",
              "      <td>0.004571</td>\n",
              "      <td>23.57</td>\n",
              "      <td>25.53</td>\n",
              "      <td>152.50</td>\n",
              "      <td>1709.0</td>\n",
              "      <td>0.1444</td>\n",
              "      <td>0.4245</td>\n",
              "      <td>0.4504</td>\n",
              "      <td>0.2430</td>\n",
              "      <td>0.3613</td>\n",
              "      <td>0.08758</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>11a</td>\n",
              "      <td>20.38</td>\n",
              "      <td>77.58</td>\n",
              "      <td>386.1</td>\n",
              "      <td>0.14250</td>\n",
              "      <td>0.28390</td>\n",
              "      <td>0.2414</td>\n",
              "      <td>0.10520</td>\n",
              "      <td>0.2597</td>\n",
              "      <td>0.09744</td>\n",
              "      <td>0.4956</td>\n",
              "      <td>1.1560</td>\n",
              "      <td>3.445</td>\n",
              "      <td>27.23</td>\n",
              "      <td>0.009110</td>\n",
              "      <td>0.07458</td>\n",
              "      <td>0.05661</td>\n",
              "      <td>0.01867</td>\n",
              "      <td>0.05963</td>\n",
              "      <td>0.009208</td>\n",
              "      <td>14.91</td>\n",
              "      <td>26.50</td>\n",
              "      <td>98.87</td>\n",
              "      <td>567.7</td>\n",
              "      <td>0.2098</td>\n",
              "      <td>0.8663</td>\n",
              "      <td>0.6869</td>\n",
              "      <td>0.2575</td>\n",
              "      <td>0.6638</td>\n",
              "      <td>0.17300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>20a</td>\n",
              "      <td>14.34</td>\n",
              "      <td>135.10</td>\n",
              "      <td>1297.0</td>\n",
              "      <td>0.10030</td>\n",
              "      <td>0.13280</td>\n",
              "      <td>0.1980</td>\n",
              "      <td>0.10430</td>\n",
              "      <td>0.1809</td>\n",
              "      <td>0.05883</td>\n",
              "      <td>0.7572</td>\n",
              "      <td>0.7813</td>\n",
              "      <td>5.438</td>\n",
              "      <td>94.44</td>\n",
              "      <td>0.011490</td>\n",
              "      <td>0.02461</td>\n",
              "      <td>0.05688</td>\n",
              "      <td>0.01885</td>\n",
              "      <td>0.01756</td>\n",
              "      <td>0.005115</td>\n",
              "      <td>22.54</td>\n",
              "      <td>16.67</td>\n",
              "      <td>152.20</td>\n",
              "      <td>1575.0</td>\n",
              "      <td>0.1374</td>\n",
              "      <td>0.2050</td>\n",
              "      <td>0.4000</td>\n",
              "      <td>0.1625</td>\n",
              "      <td>0.2364</td>\n",
              "      <td>0.07678</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  col0   col1    col2    col3     col4  ...   col25   col26   col27   col28    col29\n",
              "0  17a  10.38  122.80  1001.0  0.11840  ...  0.6656  0.7119  0.2654  0.4601  0.11890\n",
              "1  20a  17.77  132.90  1326.0  0.08474  ...  0.1866  0.2416  0.1860  0.2750  0.08902\n",
              "2  19a  21.25  130.00  1203.0  0.10960  ...  0.4245  0.4504  0.2430  0.3613  0.08758\n",
              "3  11a  20.38   77.58   386.1  0.14250  ...  0.8663  0.6869  0.2575  0.6638  0.17300\n",
              "4  20a  14.34  135.10  1297.0  0.10030  ...  0.2050  0.4000  0.1625  0.2364  0.07678\n",
              "\n",
              "[5 rows x 30 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xB6wFkArdPWN",
        "colab_type": "code",
        "outputId": "f09e45b5-4072-43c4-90f1-c2e99d3c1104",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 952
        }
      },
      "source": [
        "model = cb.CatBoostClassifier(iterations=10, cat_features=['col0'])\n",
        "cv_score = cross_validate(model, X, y, scoring='roc_auc', cv=5, return_train_score=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Learning rate set to 0.5\n",
            "0:\tlearn: 0.1914558\ttotal: 9.67ms\tremaining: 87ms\n",
            "1:\tlearn: 0.1156117\ttotal: 21ms\tremaining: 83.9ms\n",
            "2:\tlearn: 0.0817785\ttotal: 30.3ms\tremaining: 70.7ms\n",
            "3:\tlearn: 0.0636494\ttotal: 39.4ms\tremaining: 59.2ms\n",
            "4:\tlearn: 0.0519317\ttotal: 48.9ms\tremaining: 48.9ms\n",
            "5:\tlearn: 0.0443425\ttotal: 58ms\tremaining: 38.7ms\n",
            "6:\tlearn: 0.0383904\ttotal: 67.7ms\tremaining: 29ms\n",
            "7:\tlearn: 0.0359475\ttotal: 77.5ms\tremaining: 19.4ms\n",
            "8:\tlearn: 0.0318494\ttotal: 86.9ms\tremaining: 9.65ms\n",
            "9:\tlearn: 0.0263374\ttotal: 96.4ms\tremaining: 0us\n",
            "Learning rate set to 0.5\n",
            "0:\tlearn: 0.2252373\ttotal: 10.1ms\tremaining: 91.2ms\n",
            "1:\tlearn: 0.1568459\ttotal: 19.8ms\tremaining: 79.2ms\n",
            "2:\tlearn: 0.1001634\ttotal: 29.4ms\tremaining: 68.5ms\n",
            "3:\tlearn: 0.0863817\ttotal: 39.8ms\tremaining: 59.7ms\n",
            "4:\tlearn: 0.0702203\ttotal: 49.2ms\tremaining: 49.2ms\n",
            "5:\tlearn: 0.0544622\ttotal: 58.5ms\tremaining: 39ms\n",
            "6:\tlearn: 0.0457246\ttotal: 70.1ms\tremaining: 30ms\n",
            "7:\tlearn: 0.0366338\ttotal: 79.9ms\tremaining: 20ms\n",
            "8:\tlearn: 0.0315610\ttotal: 89.8ms\tremaining: 9.97ms\n",
            "9:\tlearn: 0.0240627\ttotal: 99.4ms\tremaining: 0us\n",
            "Learning rate set to 0.5\n",
            "0:\tlearn: 0.2108157\ttotal: 9.88ms\tremaining: 88.9ms\n",
            "1:\tlearn: 0.1492099\ttotal: 21.3ms\tremaining: 85.1ms\n",
            "2:\tlearn: 0.1207056\ttotal: 30.5ms\tremaining: 71.1ms\n",
            "3:\tlearn: 0.0971041\ttotal: 40.5ms\tremaining: 60.8ms\n",
            "4:\tlearn: 0.0864199\ttotal: 51.5ms\tremaining: 51.5ms\n",
            "5:\tlearn: 0.0768932\ttotal: 61ms\tremaining: 40.6ms\n",
            "6:\tlearn: 0.0703430\ttotal: 70.6ms\tremaining: 30.2ms\n",
            "7:\tlearn: 0.0589252\ttotal: 79.9ms\tremaining: 20ms\n",
            "8:\tlearn: 0.0567386\ttotal: 89.1ms\tremaining: 9.9ms\n",
            "9:\tlearn: 0.0545983\ttotal: 98.5ms\tremaining: 0us\n",
            "Learning rate set to 0.5\n",
            "0:\tlearn: 0.2234299\ttotal: 9.95ms\tremaining: 89.5ms\n",
            "1:\tlearn: 0.1389918\ttotal: 20.4ms\tremaining: 81.4ms\n",
            "2:\tlearn: 0.0905066\ttotal: 30.2ms\tremaining: 70.5ms\n",
            "3:\tlearn: 0.0653036\ttotal: 40.2ms\tremaining: 60.3ms\n",
            "4:\tlearn: 0.0598978\ttotal: 49.9ms\tremaining: 49.9ms\n",
            "5:\tlearn: 0.0490944\ttotal: 59.5ms\tremaining: 39.7ms\n",
            "6:\tlearn: 0.0446484\ttotal: 69.1ms\tremaining: 29.6ms\n",
            "7:\tlearn: 0.0388515\ttotal: 78.8ms\tremaining: 19.7ms\n",
            "8:\tlearn: 0.0339241\ttotal: 88.5ms\tremaining: 9.84ms\n",
            "9:\tlearn: 0.0304242\ttotal: 97.9ms\tremaining: 0us\n",
            "Learning rate set to 0.5\n",
            "0:\tlearn: 0.2186943\ttotal: 13.1ms\tremaining: 117ms\n",
            "1:\tlearn: 0.1407879\ttotal: 22.8ms\tremaining: 91.4ms\n",
            "2:\tlearn: 0.1008451\ttotal: 32.3ms\tremaining: 75.3ms\n",
            "3:\tlearn: 0.0750466\ttotal: 41.9ms\tremaining: 62.9ms\n",
            "4:\tlearn: 0.0666680\ttotal: 51.7ms\tremaining: 51.7ms\n",
            "5:\tlearn: 0.0511052\ttotal: 62.3ms\tremaining: 41.6ms\n",
            "6:\tlearn: 0.0468835\ttotal: 72.2ms\tremaining: 31ms\n",
            "7:\tlearn: 0.0428472\ttotal: 81.7ms\tremaining: 20.4ms\n",
            "8:\tlearn: 0.0353353\ttotal: 91.5ms\tremaining: 10.2ms\n",
            "9:\tlearn: 0.0317551\ttotal: 103ms\tremaining: 0us\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gOW_TBwsdjKg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}