57905.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "57905.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bmreiniger/datascience.stackexchange/blob/master/57905.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "caozu6iXwewN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sklearn\n",
        "import xgboost as xgb\n",
        "import joblib\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import load_breast_cancer\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H0RGbMHLwmOv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "6d74d389-5f4e-4556-d20b-6b64e36b0be0"
      },
      "source": [
        "cancer=load_breast_cancer()\n",
        "\n",
        "X_train, X_test, y_train, y_test=sklearn.model_selection.train_test_split(cancer.data, \n",
        "                                                                         cancer.target, \n",
        "                                                                         random_state=0)\n",
        "xg_clas=xgb.XGBClassifier(random_state=1, n_estimators=100).fit(X_train, y_train)\n",
        "print(xg_clas.score(X_train, y_train),\"\\n\", xg_clas.score(X_test, y_test))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.0 \n",
            " 0.9790209790209791\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AP1aLG2gzU4j",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "outputId": "480521e5-26d3-43ff-f4ac-f126f63b6fd0"
      },
      "source": [
        "xg_clas"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
              "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
              "              learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
              "              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
              "              nthread=None, objective='binary:logistic', random_state=1,\n",
              "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
              "              silent=None, subsample=1, verbosity=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1yrJ1YJx5dm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        },
        "outputId": "cff3ea40-042d-4e18-8075-f5a88b870be2"
      },
      "source": [
        "X_test[0,:].reshape(1,-1)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1.340e+01, 2.052e+01, 8.864e+01, 5.567e+02, 1.106e-01, 1.469e-01,\n",
              "        1.445e-01, 8.172e-02, 2.116e-01, 7.325e-02, 3.906e-01, 9.306e-01,\n",
              "        3.093e+00, 3.367e+01, 5.414e-03, 2.265e-02, 3.452e-02, 1.334e-02,\n",
              "        1.705e-02, 4.005e-03, 1.641e+01, 2.966e+01, 1.133e+02, 8.444e+02,\n",
              "        1.574e-01, 3.856e-01, 5.106e-01, 2.051e-01, 3.585e-01, 1.109e-01]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wsXqkKUZwnXJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "a436775c-a000-43e4-a09b-2287b5669c57"
      },
      "source": [
        "dump_list=xg_clas.get_booster().get_dump()\n",
        "num_t=len(dump_list)\n",
        "\n",
        "preds = [0]\n",
        "for n in range(num_t):\n",
        "  preds.append(xg_clas.predict(X_test[0,:].reshape(1,-1), ntree_limit=n+1, output_margin=True))\n",
        "\n",
        "print([preds[i+1]-preds[i] for i in range(num_t)])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[array([-0.19117647], dtype=float32), array([-0.17493302], dtype=float32), array([-0.1632322], dtype=float32), array([-0.15528071], dtype=float32), array([-0.14764059], dtype=float32), array([-0.134592], dtype=float32), array([-0.10049492], dtype=float32), array([-0.1302135], dtype=float32), array([-0.1268928], dtype=float32), array([-0.11612129], dtype=float32), array([-0.08275878], dtype=float32), array([-0.1172291], dtype=float32), array([-0.11398101], dtype=float32), array([-0.1115346], dtype=float32), array([-0.11064661], dtype=float32), array([-0.07345426], dtype=float32), array([-0.10717297], dtype=float32), array([-0.10513449], dtype=float32), array([-0.10398841], dtype=float32), array([-0.10136986], dtype=float32), array([-0.06175613], dtype=float32), array([-0.0963707], dtype=float32), array([-0.07870603], dtype=float32), array([-0.09415579], dtype=float32), array([-0.09520769], dtype=float32), array([-0.09691024], dtype=float32), array([-0.08932781], dtype=float32), array([0.04739833], dtype=float32), array([-0.09100223], dtype=float32), array([-0.07952356], dtype=float32), array([-0.05331874], dtype=float32), array([-0.08192921], dtype=float32), array([-0.07854772], dtype=float32), array([-0.08773494], dtype=float32), array([0.04309559], dtype=float32), array([-0.08494949], dtype=float32), array([-0.0410471], dtype=float32), array([-0.08940816], dtype=float32), array([-0.09339404], dtype=float32), array([-0.01659751], dtype=float32), array([-0.06599617], dtype=float32), array([0.02654409], dtype=float32), array([-0.08834934], dtype=float32), array([0.02391124], dtype=float32), array([-0.08487129], dtype=float32), array([-0.08129168], dtype=float32), array([-0.04954338], dtype=float32), array([-0.07444715], dtype=float32), array([-0.04855251], dtype=float32), array([-0.07878494], dtype=float32), array([-0.04307795], dtype=float32), array([-0.07436275], dtype=float32), array([-0.05560017], dtype=float32), array([-0.01397228], dtype=float32), array([-0.07300758], dtype=float32), array([-0.06667089], dtype=float32), array([-0.00923872], dtype=float32), array([-0.04245901], dtype=float32), array([-0.06061077], dtype=float32), array([-0.01046324], dtype=float32), array([-0.03374958], dtype=float32), array([-0.05404854], dtype=float32), array([-0.06557703], dtype=float32), array([-0.05135775], dtype=float32), array([0.03413153], dtype=float32), array([-0.03066111], dtype=float32), array([-0.05308104], dtype=float32), array([-0.0292182], dtype=float32), array([0.03178072], dtype=float32), array([-0.05284214], dtype=float32), array([-0.00391817], dtype=float32), array([-0.03907442], dtype=float32), array([-0.05048847], dtype=float32), array([-0.04551744], dtype=float32), array([-0.04168606], dtype=float32), array([-0.02567053], dtype=float32), array([-0.01324797], dtype=float32), array([-0.05392694], dtype=float32), array([-0.04365301], dtype=float32), array([-0.03963995], dtype=float32), array([-0.04544497], dtype=float32), array([-0.01181078], dtype=float32), array([-0.04806614], dtype=float32), array([-0.03913403], dtype=float32), array([0.00593662], dtype=float32), array([-0.00613165], dtype=float32), array([-0.03568554], dtype=float32), array([-0.03988504], dtype=float32), array([-0.03771305], dtype=float32), array([-0.00819349], dtype=float32), array([-0.04160738], dtype=float32), array([-0.03248644], dtype=float32), array([-0.00781107], dtype=float32), array([-0.02349615], dtype=float32), array([-0.03252411], dtype=float32), array([-0.03434181], dtype=float32), array([-0.01068497], dtype=float32), array([0.00462055], dtype=float32), array([-0.02988434], dtype=float32), array([-0.03110838], dtype=float32)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VIec7S_U0I7E",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "de3190d7-b102-4db0-b9dd-22e5c6d62ede"
      },
      "source": [
        "sum([preds[i+1]-preds[i] for i in range(num_t)])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-5.9249835], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hqeibgvpz1gh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "151ae97f-1f2d-4dd3-a0f3-1e0c25eebf16"
      },
      "source": [
        "xg_clas.predict(X_test[:10,:], ntree_limit=0, output_margin=True)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-5.9249835,  3.9154263,  6.179787 ,  5.1894045,  5.766832 ,\n",
              "        6.7987423,  6.1742053,  5.7832847,  5.014198 ,  7.2340755],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HdHjnxaLxtF9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "05c183d6-d785-46a0-d777-67ece2a61bba"
      },
      "source": [
        "print(dump_list[0])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0:[f7<0.0489199981] yes=1,no=2,missing=1\n",
            "\t1:[f20<16.8250008] yes=3,no=4,missing=3\n",
            "\t\t3:[f10<0.591250002] yes=7,no=8,missing=7\n",
            "\t\t\t7:leaf=0.191869915\n",
            "\t\t\t8:leaf=-0\n",
            "\t\t4:[f1<18.9599991] yes=9,no=10,missing=9\n",
            "\t\t\t9:leaf=0.120000005\n",
            "\t\t\t10:leaf=-0.13333334\n",
            "\t2:[f23<785.799988] yes=5,no=6,missing=5\n",
            "\t\t5:[f21<23.7399998] yes=11,no=12,missing=11\n",
            "\t\t\t11:leaf=0.155555561\n",
            "\t\t\t12:leaf=-0.100000001\n",
            "\t\t6:[f1<14.3000002] yes=13,no=14,missing=13\n",
            "\t\t\t13:leaf=-0\n",
            "\t\t\t14:leaf=-0.191176474\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FuhKggzs1Ij0",
        "colab_type": "text"
      },
      "source": [
        "`X_test[0,:]` appears to land in leaf 14 of the first tree, which checks out with the above predictions."
      ]
    }
  ]
}