05_Metrics_and_Scoring.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "下ごしらえ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from ipywidgets import interact"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "今回は肺がんの予測をデータセットとして使います。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "breast=load_breast_cancer()\n",
    "data=breast.data\n",
    "target=breast.target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "datasetの中身"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['target', 'data', 'feature_names', 'target_names', 'DESCR'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "breast.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(569, 30)\n",
      "(569,)\n"
     ]
    }
   ],
   "source": [
    "print(data.shape)\n",
    "print(target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "malignant: 212\n",
      "benign: 357\n"
     ]
    }
   ],
   "source": [
    "for i,j in enumerate(breast.target_names):\n",
    "    print(\"{}: {}\".format(j,sum(target==i)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "特徴量（検査値）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst radius</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>25.38</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>24.99</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>23.57</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>22.54</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   mean fractal dimension           ...             worst radius  \\\n",
       "0                 0.07871           ...                    25.38   \n",
       "1                 0.05667           ...                    24.99   \n",
       "2                 0.05999           ...                    23.57   \n",
       "3                 0.09744           ...                    14.91   \n",
       "4                 0.05883           ...                    22.54   \n",
       "\n",
       "   worst texture  worst perimeter  worst area  worst smoothness  \\\n",
       "0          17.33           184.60      2019.0            0.1622   \n",
       "1          23.41           158.80      1956.0            0.1238   \n",
       "2          25.53           152.50      1709.0            0.1444   \n",
       "3          26.50            98.87       567.7            0.2098   \n",
       "4          16.67           152.20      1575.0            0.1374   \n",
       "\n",
       "   worst compactness  worst concavity  worst concave points  worst symmetry  \\\n",
       "0             0.6656           0.7119                0.2654          0.4601   \n",
       "1             0.1866           0.2416                0.1860          0.2750   \n",
       "2             0.4245           0.4504                0.2430          0.3613   \n",
       "3             0.8663           0.6869                0.2575          0.6638   \n",
       "4             0.2050           0.4000                0.1625          0.2364   \n",
       "\n",
       "   worst fractal dimension  \n",
       "0                  0.11890  \n",
       "1                  0.08902  \n",
       "2                  0.08758  \n",
       "3                  0.17300  \n",
       "4                  0.07678  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(data,columns=breast.feature_names).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "適当にモデルを作っときます。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import cross_validation as cv\n",
    "data_train,data_test,target_train,target_test=cv.train_test_split(\n",
    "    data,target,train_size=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier as RFC\n",
    "clf=RFC()\n",
    "clf.fit(data_train,target_train)\n",
    "predict=clf.predict(data_test)\n",
    "predict_proba=clf.predict_proba(data_test)\n",
    "# precision-recall curveにはスカラー値でのスコアリングが必要"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# 今回のお題"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "簡単なところから"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.965034965034965"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 正答率\n",
    "metrics.accuracy_score(predict,target_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[52  3]\n",
      " [ 2 86]]\n"
     ]
    }
   ],
   "source": [
    "# 混同行列\n",
    "print(metrics.confusion_matrix(predict,target_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "縦軸がpredict, 横軸がtarget_test(左下がFN,右上がFP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## precision-recall関係\n",
    "> [神嶌先生の解説参照](http://ibisforest.org/index.php?F%E5%80%A4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "予測全体のPrecisionやRecall値 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "  malignant       0.96      0.95      0.95        55\n",
      "     benign       0.97      0.98      0.97        88\n",
      "\n",
      "avg / total       0.97      0.97      0.96       143\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Precision、RecallとF値とsupport(正解ラベルのデータの数)\n",
    "print(metrics.classification_report(\n",
    "        predict,target_test,target_names=breast.target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.96296296  0.96629213] [ 0.94545455  0.97727273] [ 0.95412844  0.97175141] [55 88]\n"
     ]
    }
   ],
   "source": [
    "#Precision,Recall値を再利用したい場合はこちらで抽出する\n",
    "precision, recall, fscore, support=metrics.precision_recall_fscore_support(\n",
    "    predict,target_test)\n",
    "print(precision,recall,fscore,support)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "Precision-Recall Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f72a9d575f8>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAENCAYAAAACHGKEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXGWd5/FPVVdVX6s7XJpb2oRL5MdtQdkxq+446uhK\nvGZ11pHgjAgMZB1wWHdRZ5hV1h1HYF/LEpjoCgLeMe6CzjAzDqKLiyuzKCzhFsJvIh1C7oZA0t3p\nTl9r/zinuipFkjrpOl3dVef7/id1znnq1FNPkvM7z/M7z1OpQqGAiIgkV3quKyAiInNLgUBEJOEU\nCEREEk6BQEQk4RQIREQSToFARCThFAhkXjGzZ8zsd6qUeY2ZDZhZql71mm1mttHMfjd8fZ2ZfXuu\n6yTJkZnrCkhjMLMXgOOACWAfcD9wpbsPx/k57n5OhDKbge44P7fIzK4D/hzYT/BdnwWucfdHZuPz\nDuOQE3zMLA/8BfBB4ChgJ/C3wBfd/eX6VE+aiXoEElUBeK+7dwPnA78F/MeDFWyCO/U14fc8Fvjf\nwP+c2+qUmFkWeBA4E3hXWM83AS8BS2dwvpZ4ayiNSD0CORIpAHffbmb/AJwDYGY/Ax4G3ga8Hvhn\nZvYScDPwbmAS+AbweXcvhO+5HPgU0Ae8CPyBuz9hZhuBy9z9QTN7A/AV4HRgGPiuu19jZouBjUDG\n3afM7ETgq8BvA7uB/+Lud4Sfcx1wFsEd/geBTcDF7v54tS8bnvu7wJ+Z2THuvjs85/sI7shPBtYB\nn3D3p8NjfcAtwFvC9vqeu/+JmZ0KfA04D5gCHgD+2N0HIrd+4OKwzX7H3UfCer4EfKlYwMymgCXu\n3h9ufx3Y7O6fN7O3At8B/oqg/R8I2/kad/9RWL4F2E4QaJ4wszcCN4Xt+ALw79z9oSOst8xj6hHI\nETOz1wDvAcovpn8A/BGQJ7iwfxMYBU4lCA7/KjyOmX0Y+DzBxb8b+ADBBbzSLcAqd+8BTgP+R9mx\n8qGT74efeQLwYeBLZva2suPvB+4GegiGUL4c8XvmCC68u4FXwn2vB+4ELgeOBm4D7jOzrJmlgb8j\nCFKLgIXAmvB0KYKL9QkEd/N9wH+KUo8K7wDuLwaBQ6i2bswJwIKwjlcQtM1FZceXAbvCILCQ4Dv9\nZ3c/CrgGuNfMjplB3WWeUo9AjsRfm9kEsJfg4nB92bFvuPtzAGZ2LEFPoMfdR4H9ZraK4OL5NeAy\ngrv2xwGKd64HMQYsKbsb/1VlgTAovQlY5u7jwJNmdgfwMYJhHYBfuPuPw/LfBq6u8j0/Et71dxME\ngN9z96nw2OXAV939sXD722b258AbgXHgROAzZeX/MfyOzwPPh/t2m9nNBMHwSB0DPFalTLWhuUng\nurC9MLPvAWvNrM3d9wMrgO+FZT8K/H2x/dz9f5nZYwQ3AkpoNwkFAjkSy939Z4c4trns9WIgC2w3\nMwguTCmCu3aA11C6KB7OZQRDMM+ZWT/BXenfV5Q5EXi5Imm9CfjnZds7yl4PA23h3fuFBHf0BeD/\nuPt7wzLfd/ePmdnRwL0E+ZCfl323j5nZJ8PtVPhdTyIY8tlUFgSmmdlxlIaMuoAWYCaJ3d3hd67F\nrmIQgCBImdmzwPvN7O8IemifCw8vBn7fzN4fbqcIrhsP1lgHmUcUCORIHO5Os3w4YjPBmPwxxZxA\nhc0EQz2HFd5FXwRgZr8H3BNenMttA442s0533xfuWwRsjXD+uwmGRQ51/GUzWwk8ZmbfdfedYd3/\n0t2vrywfjqUvMrP0QYLBlwgCxdnuvtfMlhOM0x+pnwJ/YWbthxkeGgY6yrZP4MBAfbC/kzUEbd0C\nrHP3jeH+zcC33H3lDOoqDUI5Aomdu+8gSIbebGZ5M0uZ2all8wPuAK4xs/MBzOy0cIjnAGb20XCY\nCYLhqALBxRRKiestBMMv15tZq5mdS9CTONywReSnmtz9nwgelf1suOtrwL81s6VhHTvN7D1m1kkw\ndLUduMHMOsL6vDl8Xx4YAgbDcfdPR61DhW8TXJzvtUDKzI4xsz8zs2VhmSeAi8wsHe57a4TzrgHe\nBXyCA4Pjdwh6Cu8Kz9dmZm81s5NmWH+ZhxQIJKrDJSAPduxjQI7gOfyXCR7BPAHA3e8B/hK428wG\ngB8SJF4rz7UMWBeWuRn4SJhzqCy3AjiFoHdwL/C5wwxhVfsuB/NfgcvN7Fh3/38EeYLVZvYy8E8E\nCWXCXsD7gdcSDINtBn4/PMcXCIar9hAkrO+dSZ3cfQx4J/Ac8BOCAPkIQe7gl2GxqwmGd14haJsf\nRjjvDuD/EuQ6vl+2fwuwHLgW2EUw7HYNunY0lVS1H6YxszuB9wE73f3cQ5S5lSA5uA/4uLs/Ee5f\nBqwi+Edzp7vfGGPdRUQkBlGi+teBCw510MzeDZzm7q8FVhI8z02YjFsdvvdsYIWZnVFzjUVEJFZV\nA4G7/4LwGepDWA58Kyz7S6DHzI4nmOW4wd03hU8orAnLiojIPBLHON9CDnwiYUu471D7RURkHpmN\nhE+jrzMjIpIoccwj2EowQaioL9yXI3ieu3J/Vbf/8KnCnqExXhnczysDo+wZ3M++/ROHfU9LOsWC\nfCsL8q0clW/jqLLXwZ+tHNUd7G9vzZBKKV6JSFOZ8UUtaiAozgw9mPuAK4HvhxNq9rj7znDRsSXh\nAmHbCWZxrojyYVd88Fx27Ro8YN/4xCR7942xd98YA0Njpdf7iq9H2Ts0xuYdgzy/Ze9hz5/LpOnu\nzNHTmQv+7Gqlp3y77HUuO7eLM/b25l/VFkmkdihRW5SoLUp6e/Mzfm/VQGBmdxOsKnmMmb0IXEdw\nt19w99vd/UfhhJpfEzw+egmAu0+a2VUEE4uKj4+un2lFs5kWju1p59ie9sOWKxQK7B+bLAsQY+wd\nGq0IGsHrF3YMMjl1+Mdn21szrwoQPV3l2610d+bo7szSktaj1SLSeKrOI5gjhXpE+alCgX0j4xVB\noxQwBvaNTu8fGh4/7IyfFNDVkT0wYIRB4sDAkaOzPUs64tCU7ngCaocStUWJ2qKktzc/60NDTSmd\nSpHvyJHvyLGw9/BlJ6emGBweZ+9QaShqoBg4hsem9788MMrWXfsOe66WdCroRXS8OkhMvw6Hq+Zp\noBaRJpLoQHAkWtJpFnS1sqCrtWrZqPmM7bv3sWnn4e9mctkWujuyDZHPEJHGpEAwC+LMZwyPTrB7\n737lM0Rk1igQzKFUKkV7a4b21gzHH91x0DLFMdAjyWfsfHl4TvIZItKYFAgaRCPkM9pyLZqfIdKA\nFAia0JzlMxpofoaIlCgQJJzmZ4iIAoFEEiWfUTQb+YxjetrpaG1RPkNkFigQSOxmI5+x65XhSOtN\ndZf1Mg7sbbQGj+EqnyHyKgoEMqei5jN6e/Ns274nWj7jpX1s2jHzfEZPRTBRPkOanQKBNAzlM0Rm\nhwKBNJ25zmccGDSUz5D5T4FAEm028hm7B0bZEnV+hvIZMg8oEIhENCvzM2aQzzjh2C5yLSnlMxrA\n+MQkw6OTDO8fZ2R0kuHR4M+R0QmG908wPDox/XpkdIK21hY+vuyMuv89KhCIzIJZzWdseOmQ51M+\nI35TUwWGRyfYNzLO0P5x9o2Ms29kovR6/wT79gf79oX7RkaDi/zE5JGvHrz0zON53ZJjZ+GbHJoC\ngcgcmkk+o6U1ywubX5nVfEb5/mbJZ0xOTQUX7YNdyEfGg4v4/gmGRsZL2yPBBT2qTEuKjrYsHW3h\n3Je24O+2o7WFjtYs7a0twfb0/tKfL+wY5Ct//Qz92wYUCETk4Ir5jN7ePB0th78wN3s+Y2Iy+H4D\no5O8uHUPA8NjDA6XLt5DZRfy4AIfDMlElWlJ09We4ejuVl7T1kVne5bOtswBf3a1Zae3u9qzdLZl\nyWXTM/7+ba3B5Xjj9oEZvb8WCgQiTWjO8xldxYUKo83PmJoqMLR/nMF9wQW9eGEf2DfG4PAYA8V9\n4fGod+m5bJrOtizH9rSXXcizdLZnyi7kWbraM+H+4MI+F7mWrvYsxx3VzsZtA0wVCnXthSkQiCTc\nrOQztkebn5Fvz7J/bILBkXGq/RhfKgX5jhxHdbeyuCNPviPL8cd2kUlBd0eWfEcuvDMv3blnM42V\nPD/1pG4eWbeT37wywglVhgrjpEAgIpEcaT5jeP/EdKA4VD5jcHic9rYsxx/dQXdHjnxnbvqinu/I\nHrDvYLmKZvvN4lNODALBxm0DCgQi0tjSqRRd7Vm62rNV52dIyakndgPQv22AN51zQt0+V8+PiYjM\nE4uO76IlnaK/zgljBQIRkXkim2nhNcd1sfk3g4xPTNXtcyMNDZnZMmAVQeC4091vrDi+ALgLOA0Y\nAS5192fDY58CLgOmgKeBS9x9LLZvICLSRE49qZsXdgyy+TdDnHpSd10+s2qPwMzSwGrgAuBsYIWZ\nnVFR7FpgrbufB1wM3Bq+9yTgk8D57n4uQeC5ML7qi4g0l1PCPEE95xNEGRpaCmxw903uPg6sAZZX\nlDkLeBDA3R042cyKKaIWoNPMMkAHsC2WmouINKFiL6B/2966fWaUQLAQ2Fy2vSXcV+5J4EMAZrYU\nWAT0ufs24CbgRWArsMfdf1prpUVEmtXxR3fQ3pqhf3v9HouN6/HRG4BbzOxxgjzAWmAyzB0sBxYD\ne4F7zOwid7+72gl7e/MxVa3xqS0CaocStUVJM7aFLTqKJzbsor2zla6O3Kx/XpRAsJXgDr+oL9w3\nzd0HgUuL22bWD/QDy4B+d3853P8D4M1A1UDQTJNEatFsE2ZmSu1QorYoada26Ovt4IkN8Ogz2zjn\nlGMivaeWgBhlaOhRYImZLTazHEGy977yAmbWY2bZ8PXlwM/dfYhgSOiNZtZmZingHcD6GddWRCQB\nTimbWFYPVQOBu08CVwEPAOuANe6+3sxWmtkVYbEzgWfMbD3B00VXh+/9FXAPwVDRkwSr394e+7cQ\nEWkixRnGG+sUCFKFais9zY1CM3b3ZqJZu75HSu1QorYoaea2+PRXHmZ8ssCqT/52pPK9vfkZL1eq\nmcUiIvNQd2crI0fwozi1UCAQEUk4BQIRkYRTIBARSTgFAhGRhFMgEBFJOAUCEZGEUyAQEUk4BQIR\nkYRTIBARSTgFAhGRhFMgEBFJOAUCEZGEUyAQEUk4BQIRkYRTIBARSTgFAhGRhFMgEBGZh3KZNBMT\nU4yOTc76ZykQiIjMQ6ec2E0B6N8++79brEAgIjIPLenrAeDXW/bM+mcpEIiIzENLFgaBYMPWvbP+\nWQoEIiLzUHdnjuOPauf5rQNMFQqz+lkKBCIi89SSvh5GRifYtmvfrH5OJkohM1sGrCIIHHe6+40V\nxxcAdwGnASPApe7+bHisB7gDOAeYCo/9MrZvICLSpJYs7OHhp3ewYete+o7rmrXPqdojMLM0sBq4\nADgbWGFmZ1QUuxZY6+7nARcDt5YduwX4kbufCZwHrI+j4iIizW5J3wJg9hPGUXoES4EN7r4JwMzW\nAMuB58rKnAVcD+DubmYnm1kvMAq8xd0/Hh6bAGb/WSgRkSZw4jEddLZl2LBldhPGUQLBQmBz2fYW\nguBQ7kngQ8DDZrYUWAT0EQwFvWRmXyfoDTwGXO3uI7VWXESk2aVTKU5b2MNTz+9mz9AoC7paZ+Vz\nIuUIIrgBuMXMHgeeBtYCk0AWOB+40t0fM7NVwJ8C11U7YW9vPqaqNT61RUDtUKK2KGn2tnidHcdT\nz+/mNwNjvPaUY2flM6IEgq0Ed/hFfeG+ae4+CFxa3DazjUA/0AlsdvfHwkP3AJ+NUrFduwajFGt6\nvb15tQVqh3Jqi5IktMWJC9oAeHz9Dk4/6dBBr5aAGOXx0UeBJWa22MxywIXAfeUFzKzHzLLh68uB\nh9x9yN13ApvN7PSw6DuAZ2dcWxGRhDnlxG5a0qlZzRNU7RG4+6SZXQU8QOnx0fVmthIouPvtwJnA\nN81sClgHXFZ2ij8BvhsGin7gkri/hIhIs8plW1h8Qp5NOwYZHZ+kNdsS+2dEyhG4+/2AVey7rez1\nI5XHy449CbyhhjqKiCTakoU99G8b4IXtA9iio2I/v2YWi4jMc68NF6CbreEhBQIRkXmuuABd/7bZ\nmYalQCAiMs91d+bIZtLsGRqdlfMrEIiIzHOpVIp8R5ahkfFZOb8CgYhIA+hqzzKoQCAiklz59iyj\nY5OMT8T/G8YKBCIiDSDfkQNgcDj+XoECgYhIA+hqzwLMSp5AgUBEpAF0dQSBQD0CEZGEyoc9gsGR\nsdjPrUAgItIAijmCIfUIRESSqZgj0NCQiEhCFXMEShaLiCTU9OOjCgQiIsnU2Rb8asDQsJLFIiKJ\nlGlJ09GaUY9ARCTJ8h1ZPTUkIpJkXeEKpIVCIdbzKhCIiDSIfHuOyakCI6MTsZ5XgUBEpEFMLzMR\nc55AgUBEpEHkZ2lSmQKBiEiDmJ5UpkAgIpJM+fbipLJ45xJkohQys2XAKoLAcae731hxfAFwF3Aa\nMAJc6u7Plh1PA48BW9z9AzHVXUQkUeasRxBexFcDFwBnAyvM7IyKYtcCa939POBi4NaK41cDzyIi\nIjNWWoq6/kNDS4EN7r7J3ceBNcDyijJnAQ8CuLsDJ5tZL4CZ9QHvAe6IrdYiIgk0lzmChcDmsu0t\n4b5yTwIfAjCzpcAioC88djPwaSDeGRAiIgkznSOIeb2hSDmCCG4AbjGzx4GngbXApJm9F9jp7k+Y\n2duAVNQT9vbmY6pa41NbBNQOJWqLkiS1RaFQoCWdYv/EVKzfO0og2Epwh1/UF+6b5u6DwKXFbTPr\nB/qBC4EPmNl7gHYgb2bfcvePVfvQXbsGI1St+fX25tUWqB3KqS1KktgWXe1ZXhnY/6rvXUtgiBII\nHgWWmNliYDvBxX1FeQEz6wGG3X3czC4Hfu7uQwRJ5GvDMm8F/kOUICAiIgeX78iye2A01nNWzRG4\n+yRwFfAAsA5Y4+7rzWylmV0RFjsTeMbM1hM8XXR1rLUUEREg6BGMjE4wMTkV2zkj5Qjc/X7AKvbd\nVvb6kcrjBznHQ8BDM6ijiIiEusJfKts3Mk5PV2ss59TMYhGRBpLviH+9IQUCEZEGMhuTyhQIREQa\nSFcYCIYUCEREkinfEf+kMgUCEZEGMhvLTCgQiIg0EOUIREQSTkNDIiIJ19UeTP9SslhEJKGymRZa\ncy3KEYiIJFm+PascgYhIkuU7sgwOj1MoxPMzLwoEIiINpqs9x8TkFKPjk7GcT4FARKTBFGcXx7Xe\nkAKBiEiDKS48F9eTQwoEIiINJu4VSBUIREQaTGloKJ5JZQoEIiINpji7WENDIiIJ1dEazC4e3j8R\ny/kUCEREGkwu2wLA+EQ8v1usQCAi0mBymeDSPTqheQQiIomUywaX7vFx9QhERBKpODQ0ph6BiEgy\n5TJhIIipR5CJUsjMlgGrCALHne5+Y8XxBcBdwGnACHCpuz9rZn3At4DjgSnga+5+ayw1FxFJqOLQ\nUN16BGaWBlYDFwBnAyvM7IyKYtcCa939POBioHixnwD+vbufDbwJuPIg7xURkSPQkk6RTqVi6xFE\nGRpaCmxw903uPg6sAZZXlDkLeBDA3R042cx63X2Huz8R7h8C1gMLY6m5iEhCpVIpctk0Y3VcfXQh\nsLlsewuvvpg/CXwIwMyWAouAvvICZnYy8DrglzOsq4iIhHKZNKMxzSOIlCOI4AbgFjN7HHgaWAtM\nhyoz6wLuAa4OewZV9fbmY6pa41NbBNQOJWqLkqS2RVtblsmpQizfP0og2Epwh1/UF+6b5u6DwKXF\nbTPbCPSHrzMEQeDb7v43USu2a9dg1KJNrbc3r7ZA7VBObVGS5LbIpFMMjIxPf/9aAkKUQPAosMTM\nFgPbgQuBFeUFzKwHGHb3cTO7HHio7M7/LuBZd79lxrUUEZEDZDPx5QiqBgJ3nzSzq4AHKD0+ut7M\nVgIFd78dOBP4pplNAeuAywDM7F8CHwWeNrO1QAG41t3vj6X2IiIJ1ZpJMzYxRaFQIJVK1XSuSDmC\n8MJtFftuK3v9SOXxcP/DQEtNNRQRkVcpX3iu+HqmNLNYRKQBlZaZqP3JIQUCEZEGVFyBNI48gQKB\niEgDKi0zoR6BiEgilRaeU49ARCSRssUeQQzrDSkQiIg0oNZMfL9JoEAgItKApp8aUo9ARCSZ4vxN\nAgUCEZEGlM0oRyAikmitMf5usQKBiEgDivN3ixUIREQaUFY5AhGRZGtVj0BEJNmmnxrSzGIRkWSa\nfmpIQ0MiIsnUqmWoRUSSTTOLRUQSTr9HICKScKUcgXoEIiKJlEqlyGXS6hGIiCRZLtuiHoGISJJl\n1SMQEUm2uHoEmSiFzGwZsIogcNzp7jdWHF8A3AWcBowAl7r7s1HeKyIiM9OaSbN3qA49AjNLA6uB\nC4CzgRVmdkZFsWuBte5+HnAxcOsRvFdERGYgl22p2zyCpcAGd9/k7uPAGmB5RZmzgAcB3N2Bk82s\nN+J7RURkBrKZNFOFAhOTtQWDKIFgIbC5bHtLuK/ck8CHAMxsKbAI6Iv4XhERmYHWmGYXR8oRRHAD\ncIuZPQ48DawFahq46u3Nx1GvpqC2CKgdStQWJUlui3xXa/BnT3tN54kSCLYS3OEX9YX7prn7IHBp\ncdvMNgL9QEe19x7Krl2DUYo1vd7evNoCtUM5tUVJ0ttiajK4396+c4Cju9tmfJ4ogeBRYImZLQa2\nAxcCK8oLmFkPMOzu42Z2OfCQuw+ZWdX3iojIzJQWnqvtyaGqOQJ3nwSuAh4A1gFr3H29ma00syvC\nYmcCz5jZeoInhK4+3HtrqrGIiADx/UpZpByBu98PWMW+28peP1J5/HDvFRGR2hV/pWy8xh+n0cxi\nEZEGVVyBdLTGHoECgYhIg6pbjkBEROan0s9VKhCIiCRSXD9Oo0AgItKgcjE9NaRAICLSoFqz8fxu\nsQKBiEiD0tCQiEjC6akhEZGEmw4E6hGIiCRTazg0NK4egYhIMmXDHsGoegQiIsmUy+ipIRGRRMvp\n8VERkWRrSadpSacY19CQiEhy5bItWn1URCTJctm0Fp0TEUmy1kyLhoZERJIsm00rWSwikmS5jHIE\nIiKJ1ppNMzGpQCAikljZ8DcJaqFAICLSwIqTymqhQCAi0sByMfQIMlEKmdkyYBVB4LjT3W+sON4N\nfAdYBLQAN7n7N8JjnwIuA6aAp4FL3H2s5pqLiEh9egRmlgZWAxcAZwMrzOyMimJXAuvc/XXA24Gb\nzCxjZicBnwTOd/dzCQLPhTXXWkREgHh6BFFCyVJgg7tvcvdxYA2wvKJMAciHr/PAbnefCLdbgE4z\nywAdwLaaay0iIkD9cgQLgc1l21vCfeVWA2eZ2TbgSeBqAHffBtwEvAhsBfa4+09rrbSIiASKv1JW\ni0g5ggguANa6+++a2WnAT8ysOBS0HFgM7AXuMbOL3P3uaifs7c1XK5IYaouA2qFEbVGS9LY4ekF7\nzeeIEgi2EiSBi/rCfeUuAa4HcPfnzWwjcAZwMtDv7i8DmNkPgDcDVQPBrl2DEarW/Hp782oL1A7l\n1BYlagsYG52oXqiKKIHgUWCJmS0GthMke1dUlNkEvBN42MyOB04H+gmGnt5oZm3AKPCO8HwiIhKD\n4q+U1aLqGdx9ErgKeABYB6xx9/VmttLMrgiLfRF4s5k9BfwE+Iy7v+zuvwLuAdYS5A5SwO0111pE\nRIA65gjc/X7AKvbdVvZ6O0Ge4GDv/QLwhRrqKCIih1CXHoGIiMxfcfQIFAhERBqY1hoSEUm4es0s\nFhGReUo9AhGRhFOPQEQk4dQjEBFJOPUIREQSLtOSIpWq7RwKBCIiDSyVStU8l0CBQESkwdU6u1iB\nQESkwdWaJ1AgEBFpcLU+OaRAICLS4NQjEBFJOPUIREQSTk8NiYgknJ4aEhFJOPUIREQSTj0CEZGE\nU49ARCTh1CMQEUm43qPaa3q/AoGISIN763kn1fT+TJRCZrYMWEUQOO509xsrjncD3wEWAS3ATe7+\njfBYD3AHcA4wBVzq7r+sqdYiIjItVeM61FV7BGaWBlYDFwBnAyvM7IyKYlcC69z9dcDbgZvMrBhk\nbgF+5O5nAucB62uqsYiIxCpKj2ApsMHdNwGY2RpgOfBcWZkCkA9f54Hd7j4R9hTe4u4fB3D3CWAg\nprqLiEgMogSChcDmsu0tBMGh3GrgPjPbBnQBHwn3nwK8ZGZfJ+gNPAZc7e4jNdVaRERiE1ey+AJg\nrbufBLwe+LKZdREEmvOBL7v7+cAw8KcxfaaIiMQgSo9gK0ESuKgv3FfuEuB6AHd/3sw2AmcQ9CQ2\nu/tjYbl7gM9G+MxUb2++eqmEUFsE1A4laosStUXtovQIHgWWmNliM8sBFwL3VZTZBLwTwMyOB04H\n+t19J7DZzE4Py70DeDaWmouISCxShUKhaqHw8dFbKD0+eoOZrQQK7n67mZ0IfAM4MXzL9e7+vfC9\n5xE8PpoF+oFL3H1v7N9ERERmJFIgEBGR5qWZxSIiCadAICKScAoEIiIJF2mtodlQbf2isMytwLuB\nfcDH3f2J+tayPiKs5XQRpcduB4FPuPvT9a1lfUT5dxGWewPwj8BH3P0Hdaxi3UT8P/I24GaChzF2\nufvb61rJOqllvbNmYmZ3Au8Ddrr7uYcoc8TXzTnpEURZv8jM3g2c5u6vBVYCX617Resg4lpO/cDv\nuPt5wBeBr9W3lvURsS2K5W4AflzfGtZPxP8jPcCXgfe5+znAh+te0TqIYb2zZvJ1gnY4qJleN+dq\naGh6/SJ3HweK6xeVWw58CyBcrbQnnKPQbKq2hbs/UvbI7SMEy340oyj/LgA+STA58Tf1rFydRWmL\ni4B73X0rgLu/VOc61kuUtjjoemd1rGNduPsvgFcOU2RG1825CgQHW7+o8uJWWWbrQco0gyhtUe6P\ngH+Y1RrNnaptYWYnAf/a3f87UNvau/NblH8XpwNHm9nPzOxRM/vDutWuvqK0xWrgrHC9syeBq+tU\nt/lmRtdNJYsbiJm9nWA5jyjLdDSrVRz4/Zs5GFRTXMvr3cAy4HNmtmRuqzRnDrXemUQwV4EgyvpF\nW4HXVCk0kYl9AAABYklEQVTTDKK0BWZ2LnA78AF3P1zXsJFFaYvfAtaE61n9G4L/8B+oU/3qKUpb\nbAF+7O773X038HOCVX6bTdT1zn4AwXpnQHG9s6SZ0XVzrpIp0+sXAdsJ1i9aUVHmPoIE0PfN7I3A\nnnDtomZTtS3MbBFwL/CH4T/yZlW1Ldz91OLrcHnzv3X3yrWvmkGU/yN/A/yVmbUArcC/AP5bXWtZ\nH1Haorje2cPl653VtZb1k+LQPeEZXTfnpEfg7pPAVcADwDpgjbuvN7OVZnZFWOZHwEYz+zVwG/DH\nc1HX2RalLYDPAUcDXzGztWb2qzmq7qyK2BblmnZ9lIj/R54jeHLqKYKHCG5396Zb1DHiv4svAm82\ns6eAnwCfcfeX56bGs8fM7iZ4bPp0M3vRzC6J47qptYZERBJOyWIRkYRTIBARSTgFAhGRhFMgEBFJ\nOAUCEZGEUyAQEUk4BQIRkYRTIBARSbj/D2L6ivDPvD71AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f72ac2b7278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# precision-recall curve\n",
    "precision,recall,thresholds=metrics.precision_recall_curve(target_test,predict_proba[:,1])\n",
    "plt.plot(recall,precision) # recallが先なのを注意\n",
    "plt.title(\"Precision-Recall Curve\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98935439073855747"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AUC算出\n",
    "metrics.auc(recall,precision) # こちらもrecallが先"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "precision_recall_curve関数の返り値の中身"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.864078</td>\n",
       "      <td>0.926316</td>\n",
       "      <td>0.956044</td>\n",
       "      <td>0.966292</td>\n",
       "      <td>0.977273</td>\n",
       "      <td>0.988372</td>\n",
       "      <td>0.987805</td>\n",
       "      <td>0.986301</td>\n",
       "      <td>0.985294</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.988764</td>\n",
       "      <td>0.977528</td>\n",
       "      <td>0.966292</td>\n",
       "      <td>0.966292</td>\n",
       "      <td>0.955056</td>\n",
       "      <td>0.910112</td>\n",
       "      <td>0.808989</td>\n",
       "      <td>0.752809</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0         1         2         3         4         5  \\\n",
       "precision  0.864078  0.926316  0.956044  0.966292  0.977273  0.988372   \n",
       "recall     1.000000  0.988764  0.977528  0.966292  0.966292  0.955056   \n",
       "\n",
       "                  6         7         8    9  \n",
       "precision  0.987805  0.986301  0.985294  1.0  \n",
       "recall     0.910112  0.808989  0.752809  0.0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(np.c_[precision,recall],columns=[\"precision\",\"recall\"]).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "recallが閾値以下になった時のprecisionのカットオフにするとかなら"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "criteria= 0.986\n"
     ]
    }
   ],
   "source": [
    "for i,j in enumerate(recall):\n",
    "    if j<0.8 and i==0:print(\"error\");break\n",
    "    elif j<0.8:\n",
    "        print(\"criteria= {0:.3}\".format(precision[i-1]))\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# みんな大好きROC curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "ROC curveも関数が用意されています"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "fp_ratio, tp_ratio, thresholds = metrics.roc_curve(\n",
    "    target_test,predict_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f72a97a8a90>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEDCAYAAAA2k7/eAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE0NJREFUeJzt3W2MnWWdx/Hv6cx0OkApTwMrrbCrwB/FAOsDGrNxYTVp\n2RDri3WlJG5ko7CuGN/pK7JvSFYSUaL4AIZo1GxqIiZiooAvfFg1KCYILsI/ICxC6cLw0Fra6cyZ\nOWdfzCND6dzTzv3Qub+fZMK5z1w9c82VM7/z57quc51Ov99HkrT2rau7A5Kkahj4ktQSBr4ktYSB\nL0ktYeBLUksY+JLUEoPLNYiI24ErgGcz88LXaPNF4HJgP/CRzPz9qvZSknTUilT43wC2vtY3I+Jy\n4I2ZeS5wLfC1VeqbJGkVLRv4mflL4KXDNNkOfGu27W+ATRFxxup0T5K0WlZjDn8z8NSi612z90mS\nGsRFW0lqiWUXbQvYBbx+0fWW2fsOq9/v9zudzir8+Or1en3GJ6Y4cHCKAwe77D/YXXR7igPjM/eN\nH5ya/97+g10OjE9xYKLL/vGZttO9lZ9jNDS4juM3DHHchsHZryGOH5m5Pn7DECMbBlk/OFDCby2p\nCf75fecdcXAWDfzO7Neh3Al8AvhuRLwL2JOZzy77gJ0OY2P7Cv74avT7ff5yoMvYS+M8t+cAY3sO\n8txL47ywd5z9E1OMz34dnJjmSI6cG14/wMj6AU4YGeT0kzYwMjzIyPAgJ28aodPvzV8fNzzIhvWD\nHDc8wMiGmftG1s/8d2hwbf9P2ejoxsY9L+riWCxwLFZHkW2Z/wVcCpwaEX8G/gNYD/Qz87bM/FFE\n/GNEPMbMtsyry+zw0Zqa7vH83oOM7RnnuZfGGduz+OsgE93pV/2bDsyH8aknjnDc8AAbZoN5ZP5r\nYNHtudAemGmzYSaw16079GumT2ZJVVg28DPzqgJtrlud7qye5/eM8/juv8yH+Uy4H+TFfQc51InQ\nw+sHOP3kEUZPGuH0k0YYPWkDo7PXp564gcGBtV1ZS1r7VmMOvzGmpns88Njz/Oz+XTz0v6/eSXry\nxmHO3bxpPshngn2E0ZNH2DgyxLG6piBJRayJwH9h70F+/sAz/PeDz7D35UkAzt2yibedN8rppxzH\n6SeNcNqmDawfcjFTUns1PvAnutP89uFn+d0jY0weYn69O93jid1/od+fmWd/79u2cOnFZ7J59IQa\neitJzdXYwN819jI/+/0z/Pp//o/xianDtn3DmSfy9xefySVvOoNhq3hJOqRGBX6v1+c3Dz/Lz+7f\nxaNP7wVg0wnree/b/pr3XPQ6Tts0UnMPJenY1ajA//kDz/DtuxOAC/7mFC69+EwuOuc0d8hI0ipo\nVODvH+8C8G/bL+CSN3n+miStpkaWzsdtaNTrkCStCY0MfEnS6jPwJaklDHxJagkDX5JawsCXpJYw\n8CWpJQx8SWoJA1+SWsLAl6SWMPAlqSUMfElqCQNfklrCwJekljDwJaklDHxJagkDX5JawsCXpJYw\n8CWpJQx8SWoJA1+SWsLAl6SWMPAlqSUMfElqCQNfklrCwJekljDwJaklDHxJagkDX5JaYrBIo4jY\nBtzMzAvE7Zl545Lvnwh8BzgLGABuysxvrm5XJUlHY9kKPyLWAbcAW4ELgB0Rcf6SZp8AHsrMi4HL\ngJsiotCLiSSpGkWmdC4BHs3MJzOzC+wEti9p0wc2zt7eCLyQmVOr101J0tEqEvibgacWXT89e99i\ntwBvjohngAeAT61O9yRJq2W1pl22Avdn5j9ExBuBn0TEhZn58uH+0ejoxldcH3/8MACbNh33qu+t\ndW37fQ/HsVjgWCxwLI5ekcDfxcxi7Jwts/ctdjXwnwCZ+aeIeAI4H/jd4R54bGzfK673758AYO/e\nA6/63lo2OrqxVb/v4TgWCxyLBY7FgqN54SsS+PcB50TE2cBu4Epgx5I2TwLvA34VEWcA5wGPH3Gv\nJEmrbtk5/MycBq4D7gEeAnZm5sMRcW1EXDPb7Abg3RHxIPAT4NOZ+WJZnZYkrVyhOfzMvAuIJffd\nuuj2bmbm8SVJDeU7bSWpJQx8SWoJA1+SWsLAl6SWMPAlqSUaFfj9ujsgSWtYowJ/aqoHwNBAo7ol\nSWtCo5K1Oxf4gwM190SS1p5GBf7E1DQA64ca1S1JWhMalazd7kyFv36wUd2SpDWhUck6OV/hO6Uj\nSautWYFvhS9JpWlUsnat8CWpNI0K/ImpHp0ODKzr1N0VSVpzGhX43W6P9YMDdDoGviSttkYF/uTU\ntFsyJakkjUrXyW7PBVtJKkmj0nWmwnfBVpLK0LDA7zFkhS9JpWhMuvb7fSa7VviSVJbGBP50r0+/\n75uuJKksjUnXhXfZWuFLUhmaE/ielClJpWpMuk7On4XfmC5J0prSmHSd7HqOjiSVqTGBP/dpVy7a\nSlI5GpOu8xW+i7aSVIrmBP5che+irSSVojHpaoUvSeVqTuDP7dKxwpekUjQmXecWbYet8CWpFI0J\n/InZKR334UtSORqTrl0XbSWpVI1JVxdtJalczQl8F20lqVSDRRpFxDbgZmZeIG7PzBsP0eZS4AvA\nEDCWmZetpCNzFb6LtpJUjmXL6YhYB9wCbAUuAHZExPlL2mwCvgxckZlvAT640o5Y4UtSuYqk6yXA\no5n5ZGZ2gZ3A9iVtrgLuyMxdAJn5/Eo74hy+JJWryJTOZuCpRddPM/MisNh5wFBE/BQ4AfhiZn57\nJR1xl44klWu10nUQeCtwObANuD4izlnJA8yfpWOFL0mlKFLh7wLOWnS9Zfa+xZ4Gns/Mg8DBiPgF\ncBHw2OEeeHR04/ztPtDpwOv+6kQ6nU6Bbq0ti8ei7RyLBY7FAsfi6BUJ/PuAcyLibGA3cCWwY0mb\nHwBfiogBYBh4J/D55R54bGzf/O39412GBtfx/PMvF+z62jE6uvEVY9FmjsUCx2KBY7HgaF74lp3S\nycxp4DrgHuAhYGdmPhwR10bENbNtHgHuBh4E7gVuy8w/rqQjk91pp3MkqUSF9uFn5l1ALLnv1iXX\nnwM+d6Qd6U71XLCVpBI1JmGt8CWpXM0J/Kmen2crSSVqTMJOdnusH7LCl6SyNCLwp6Z79Pp9z8KX\npBI1ImHnP+3KCl+SStOIwJ/0064kqXSNSNiFYxUa0R1JWpMakbDzJ2U6pSNJpWlG4M+dhW+FL0ml\naUTCWuFLUvkaEfhd5/AlqXSNSNiJ7tyHn1jhS1JZGhH43am5jzdsRHckaU1qRMJO+vGGklS6RiSs\nH2AuSeVrROB33ZYpSaVrRMJOuC1TkkrXiMB3W6Ykla8RCTvptkxJKl0zAt9tmZJUukYk7MK2TCt8\nSSpLMwLf8/AlqXSNSNiFT7xqRHckaU1qRMJOdqfpAIMDjeiOJK1JjUjYyakeQ4Pr6HQ6dXdFktas\nxgS+C7aSVK5mBH532gVbSSpZI1LWCl+SyteIwO9OTfumK0kqWSNSdrLb8yx8SSpZ7Sk73esx3et7\nFr4klaz2wJ8/OM0pHUkqVe0pO3eOzpCLtpJUqtoDv9v1pExJqkLtKTvhSZmSVInaA7/rWfiSVInB\nIo0iYhtwMzMvELdn5o2v0e4dwK+BD2Xm94s89sKnXRn4klSmZVM2ItYBtwBbgQuAHRFx/mu0+yxw\n90o6MPdpV0Nuy5SkUhUpqy8BHs3MJzOzC+wEth+i3SeB7wHPraQD3dkKf9gpHUkqVZGU3Qw8tej6\n6dn75kXEmcAHMvOrwIrOOJ6Yq/BdtJWkUq1WWX0z8JlF14VDv+sbrySpEkUWbXcBZy263jJ732Jv\nB3ZGRAc4Dbg8IrqZeefhHnh0dCPrNwwBcNqpxzM6urFwx9eaNv/uSzkWCxyLBY7F0SsS+PcB50TE\n2cBu4Epgx+IGmfmGudsR8Q3gh8uFPcDY2D5e3HMAgPEDk4yN7VtB19eO0dGNrf3dl3IsFjgWCxyL\nBUfzwrfsPEpmTgPXAfcADwE7M/PhiLg2Iq45xD/pr6QDky7aSlIlCu3Dz8y7gFhy362v0fZfV9KB\nSRdtJakStZfVnpYpSdWoPWXnj1awwpekUtUe+HPHI1vhS1K5ak/ZhbN0rPAlqUz1B76nZUpSJWpP\n2bkKf8jAl6RS1Z6y3alphgbX0ems6AgeSdIK1R74k92e0zmSVIHak3ZyatoFW0mqQP2Bb4UvSZWo\nPWknp3p+2pUkVaD2wO9OTTPs59lKUulqTdper8/UdN8tmZJUgVqTdtJzdCSpMvUGvidlSlJlGlHh\nu2grSeVrRIXvoq0kla/WpO1OzZ2jY4UvSWWrNfAnunOLtlb4klS2RlT4LtpKUvkasWjrtkxJKl8j\nFm2t8CWpfFb4ktQSjajwPVpBksrXkEVbK3xJKlvNFb7bMiWpKjXP4VvhS1JVrPAlqSUaUeG7aCtJ\n5at50Xamwh92W6Yklc5tmZLUEo2Y0vGNV5JUvkYs2lrhS1L5aq/wBwfWsa7TqbMbktQKtZ+l46dd\nSVI16t2l0+05nSNJFRks0igitgE3M/MCcXtm3rjk+1cBn5m93Ad8PDP/sNzjTkxNuyVTkiqybHkd\nEeuAW4CtwAXAjog4f0mzx4H3ZOZFwA3A14v88G6351n4klSRIhX+JcCjmfkkQETsBLYDj8w1yMx7\nF7W/F9hc5IdPTvXckilJFSlSXm8Gnlp0/TSHD/SPAj9e7kGne32mpq3wJakqhebwi4qIy4Crgb9b\nrm13dg/+CccPMzq6cTW7cUxyDBY4FgsciwWOxdErEvi7gLMWXW+Zve8VIuJC4DZgW2a+tNyDTswG\nfr/XY2xsX6HOrlWjoxtbPwZzHIsFjsUCx2LB0bzwFQn8+4BzIuJsYDdwJbBjcYOIOAu4A/hwZv6p\nyA+eC3yndCSpGsumbWZOA9cB9wAPATsz8+GIuDYirpltdj1wCvCViLg/In673ONOTPoB5pJUpUJz\n+Jl5FxBL7rt10e2PAR9byQ/2HB1JqlZtaTs3peMbrySpGrUFvhW+JFWrxsD3A8wlqUr1TelM+gHm\nklSl2ufwrfAlqRr1B74VviRVwkVbSWqJBlT4TulIUhVqr/A9WkGSqlH/Lh0XbSWpEg2Y0rHCl6Qq\nNGBKxwpfkqpQe4U/ZIUvSZWo/WiFYSt8SapE7Yu2VviSVI1a5/AHBzqs63Tq6oIktUqtc/gu2EpS\ndWoNfKdzJKk6tc7hu2ArSdWpdQ7fCl+SqlNr4DuHL0nVqS/wp3oenCZJFao1cT0aWZKqU2/gW+FL\nUmVqTVwXbSWpOjVX+E7pSFJVnNKRpJZw0VaSWsIKX5Jawgpfklqi3l06VviSVJmaK3wDX5Kq4rZM\nSWoJF20lqSVctJWklhgs0igitgE3M/MCcXtm3niINl8ELgf2Ax/JzN8v97gu2kpSdZZN3IhYB9wC\nbAUuAHZExPlL2lwOvDEzzwWuBb5W5IcPW+FLUmWKlNiXAI9m5pOZ2QV2AtuXtNkOfAsgM38DbIqI\nM5Z7YCt8SapOkcTdDDy16Prp2fsO12bXIdq8itsyJak6LtpKUksUWbTdBZy16HrL7H1L27x+mTav\n8MObtneKdLAtRkc31t2FxnAsFjgWCxyLo1ekwr8POCcizo6I9cCVwJ1L2twJ/AtARLwL2JOZz65q\nTyVJR2XZwM/MaeA64B7gIWBnZj4cEddGxDWzbX4EPBERjwG3Av9eYp8lSUeg0+/36+6DJKkCbpOR\npJYw8CWpJQx8SWqJQmfpHI2yzuE5Fi03FhFxFfCZ2ct9wMcz8w/V9rIaRZ4Xs+3eAfwa+FBmfr/C\nLlam4N/IpcAXgCFgLDMvq7STFSnwN3Ii8B1mtooPADdl5jer7mfZIuJ24Arg2cy88DXarDg3S63w\nyzyH51hTZCyAx4H3ZOZFwA3A16vtZTUKjsVcu88Cd1fbw+oU/BvZBHwZuCIz3wJ8sPKOVqDg8+IT\nwEOZeTFwGXBTRJReuNbgG8yMwyEdaW6WPaVT2jk8x6BlxyIz783MvbOX91LgeIpjVJHnBcAnge8B\nz1XZuYoVGYurgDsycxdAZj5fcR+rUmQs+sDcO7A2Ai9k5lSFfaxEZv4SeOkwTY4oN8sO/NLO4TkG\nFRmLxT4K/LjUHtVn2bGIiDOBD2TmV4G1/K7sIs+L84BTIuKnEXFfRHy4st5Vq8hY3AK8OSKeAR4A\nPlVR35rmiHLTRdsGiojLgKtZmM9vo5t55e+/lkN/OYPAW5mZr90GXB8R59TbpdpsBe7PzDOBvwW+\nHBEn1NynY0bZgV/KOTzHqCJjQURcCNwGvD8zD/e/dMeyImPxdmBnRDwB/BMzf9jvr6h/VSoyFk8D\nd2fmwcx8AfgFcFFF/atSkbG4Gvg+QGb+CXgCeNX6TwscUW6Wvdgxfw4PsJuZc3h2LGlzJzMLMd9d\n4+fwLDsWEXEWcAfw4dkn81q17Fhk5hvmbkfEN4AfZubSM5zWgiJ/Iz8AvhQRA8Aw8E7g85X2shpF\nxuJJ4H3Ar2bnrM9jZrPDWtThtf/P9ohys9QK33N4FhQZC+B64BTgKxFxf0T8tqbulqrgWCy2Zs//\nKPg38ggzO5UeZGYx/7bM/GNdfS5LwefFDcC7I+JB4CfApzPzxXp6XJ6I+C9mtiOfFxF/joirVyM3\nPUtHklrCRVtJagkDX5JawsCXpJYw8CWpJQx8SWoJA1+SWsLAl6SWMPAlqSX+HzfId2asgJBLAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f72ac2e0ba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fp_ratio,tp_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98491468997086973"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.auc(fp_ratio,tp_ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "ROCカーブのAUCは専用の関数があります"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98491468997086973"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.roc_auc_score(target_test,predict_proba[:,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "あとで使うので、この入出力の形式を覚えておいてください"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "## おまけ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "scrolled": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAENCAYAAAAMmd6uAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHDZJREFUeJzt3XuYXFWd7vFv9SUXOh0OgQZCQoKSyS+Kw00JCgpEPRNQ\nNOrxQpgBZEbJDMZxbsJcDsM8yvHIPINGDCowGRAYDONEEc4oAR2GO4hjEA3wM0ASkpB7IElXupPq\n7jp/rF2potKXne667Or9fp6Hh967VlWtWql+96q1Vq3O5PN5RERk9GuqdwVERKQ2FPgiIimhwBcR\nSQkFvohISijwRURSQoEvIpISCnwRkZRQ4IuIpERLvSuQNma2BjgS6AE6geXA59x9T0mZM4AvA6cB\nvcDDwF+7+/MlZdqjMh8FDgM2A/cC17j7jgGe+0+BzwJvAnYATwBfcveVFX2RNWJmq4E/cvf/rHdd\nBmJmXwI+ArwF+LK7f6ns9guBrwCHAw8Af+jur9e8osNgZu8DFgPHAk8Bl7r7KwOUnQXcALwd2AJc\n4e53R7ddCNwIFL4F2gyMB97u7ivM7Grg74BuIBOVO9Hd11TppY1a6uHXXh74oLtPBE4GTgH+pnCj\nmb2LcBH4ITCZEM7PAo+Z2XFRmVbgPwkh8nvRY70L2AbM7u9Jzex64PPAQsIFYiZwN/DBg30BZtZ8\nsPeph4TUcxXwReD/ld9gZicA3wF+HzgK6AK+XdPaDZOZHQ4sIwTxJOC/gbsGKNsM/Ai4h/DeWwDc\nYWYzANz9Tndvd/eJ0Xv5cuAld19R8jBLo9sL5dZU67WNZurh10cGwN23mNlyQvAXXAvc6u6LS85d\nZWZvB/4B+DRwCTAVOMvdu6LH2kboKR4g+sW6HDjd3f87Op0DvldS5kHgdnf/l+j4EuAz7v6e6LiP\ncLH4M6A5qnfW3b9Y8hh3A//l7ovMbDLwTeAsYDewyN2/OUD9rgbeSujBfRRYC1zi7r/sv/nAzG4D\npgH3mlkv8CXg+8Bq4DPA1dHP55jZO4HroudYA/yZuz8UPc5E4GvABwifpm4F/t7d82Z2PLCE8O+z\nD/iZu88fqE79cffbo+f5g35uvhC4x90fi8pcBTxvZm3unjWzK4Ez3f3DA7TBlYRPbEcCrwD/u6TX\nfDUww90vio6nR+3R4u59ZnZY1CZzgXHAQ+7+sYN4aR8DfuPuP4ge/x+AbWY2091/W1Z2FjDZ3b8R\nHT9oZo8BFxH+ncpdAtx2EHWRmBT4dWRmU4HzgJ9Gx+OBM4Cr+in+b8D/iX5+H3BfIexjeB+wriTs\n4yrfaGkeYZipm/BJ4g5C7xUz+x/A7wELzCxDGF76IfApwkf+n5rZC+7+gJmdCdzr7pNKHvtDhLD/\nNOF13kD41NIvd7/YzN5DGAJ5MKrD9Ojmswgh02dmxxB617/v7sujYYhlZmbuvh34LrAReDMwISr7\nCnAzYchsubufY2ZjgHcUnt/MfhW9LigOMxT+f6e7Lxy4Wfc7AXis5DW9bGZ7CZ++Vrj7tUPc/0XC\nBWGzmX2C0Gs+3t03R7eX//uVHt8B7CJ8SswS3neY2bGET5SF1wNvfG2Xu/vSqO6/Kqn7HjN7MTpf\nHvj9yQBvKz8Z/Ru+B7i07KYPmdk2wr/VDe7+nRjPIWUU+PVxt5lBCJifEXruED4aNxHe1OU2AkdE\nPx8O/OIgnu/wAR7zYH3F3XdGPz9iZnkze7e7Pwp8HHg8Cp/TgSPcvXCBWmNm/wxcADwQ9WgnlT32\no+6+HMDMbge+ELNOmbLjPHB14WIY9az/o/DY7v4zM/sF8IHoU8p5wKHuvhfoNrNFhF7zzYRPQdPN\nbIq7bwAeLzyJu58Us36DmQDsLDu3C2iPc2d3X1by8/fN7G8JF+J7B7tf9OlrLjDJ3XdFpx+JHmcd\nYdglTt23xKy7A1vM7K+ARcB7gbMJw5LlLgYecfe1JefuIozxbwbeSbhgv+bu/Q4hycAU+PUxz90f\njHqodxKCfBfwGtBHGLsv7yVNJozRA2yPjuM62PIDWV92fBcwH3iUMDxxe3R+GjDFzAqTxxnChezh\nQR57U8nPe4BxZtbk7n0jrOd04JNm9qGSurQQwmY60ApsjC7Amei/wsTjF4FrgJ9Hr+Vr7n7LMOoz\nkE5gYtm5QwlDYEMys4uBPweOi061UewUDGYqsKMk7Icjdt3dvcfMPkKY4L2S0Fm5C9jbz+NeRGjz\n0vu/UHL4hJl9g9DBUOAfJAV+fRTG8B8xs+8SxlI/Gn0sfgL4BPBQ2X0+STT0E/3/y2Y2Puawzs+A\nxWZ26iDj4lngkJLjo/spUz5E8D1guZldC5xOWI0CsA542d0tRt1GYqC9vUvPrwNuc/cF5YXM7GjC\n8NTh7n7AY7n7FuCyqOyZhGGph6Khl98QLmylCsMed7j75THqvxLY/0khmjNoJcaQiJlNA24C5rj7\nE9G5FRQ/8ZT/e5Ze8NcBk8xsYnnoR0M6z3Fg2xZe2wJ3/15U90tK7tcGHB+dP4C7/wY4p6T8Y4T5\nktLnPjOq5zIGVzrcJAdBgV9/iwhDHr/r7r8G/hq4z8xeIPxCtAJ/Sfgoe1p0n9sJQbTMzP6cEBCT\nonMr3P2+0idw9xfN7FvA98zsMsLQRBMhoKe7+z8CzwAfM7MlwBTgj3hjr/sA7v6MmW0H/pkwp1AI\nj58Du83sCuB6wtDILGC8u8cdiorzC72JMPZeOjRQfr87CD30ZYQL5RjCxWmVu79qZvcDX48mTDsJ\nq6KmuvvDZvZx4IloOOd1wqevvui1HzD+3B8zayH8njUBrWY2FshFn1z+FXg8CrpnCBPPy9w9G933\nauBsd39vPw/dFtVlm5k1EcK3tE7PAFdEAb6L8L4iqvsmM/sJ8C0zWxi97ne5+yPRkE6cIaUfAv9o\nZh8FfkyYfH2mnwnbQjv8LuF92kxYQHA0ZYEfvYb9r7/kvh8GHnb3181sNmG478oYdZQyQy7LNLMl\nZrbZzJ4dpMz1ZrbKzJ4xs5MHKidAWc8pWl3zXeDvo+PHCOOr/4sw7r6a0As8091fisrsA94PvEBY\nu70TeJIwVv9Uf0/q7l8gfKS+gTB09CIh8AvjvV8nBPMm4BZCUA5Y7xJ3EiaF/7XkufqA8wmrW1YT\nxnpvJhoCMLN3m9lQwwlx/jLPVwkrmHaY2V/0dz93X0+YbP5bYCthBdBfUXzvX0y4CDxH+G7C9yl+\nujkNeCqq693Anw5jOeDNhCGqC6I67AH+IKrbc8AfE9pwE2Ht+edK7nssJZO6Za/recInwyej+55A\nGFor3P5TwpDHs8DTHDiufxHhuyAvEMbG486ZFB5/G+E9+hVCu70jeo0AmNnfmNl/lD3fxqiuc4D/\n6e65kvJjCcM0t/bzdBcAL0b/DrcS5pLK358SQ2aov3hlZu8m9ABuc/cT+7n9PGChu38wmqz7hru/\nsyq1FUkRM/sl8D53f63edZHRYcgefrQCY7A33DyiNbPu/hRwqJkdVZnqiaSXu5+qsJdKqsQY/hTC\nJFDBhujc5v6Li8QzwARiYfLwrdFwjYjEpElbSayDmEAUkRgqEfgbKH7jEMIa3w1D3Smfz+czmWSt\nrOrry7Pit1v4yeNrePq5TfTlYfzYFs46ZQpHHnbI0A8gIlJln3z/zGEHZ9zAL3whpT/3EFYW3BXt\nWfJ6yVe7B37ATIatW2N9v6RiNm7P8tAzr/Lyxv4XiLy2q5vtu8J3QaYf3c6cU6Yw+y1HMm5MdT8I\ndXS017wtkkptUaS2KFJbVMaQSWZmdxK+MHG4mb1CWG87Bsi7+03u/mMz+0C0j0aWA/fAqKtcTx+/\n/O1W/mvFBnxd2HU2Q7jglBs7pomzTprM2SdP4U2Ty79EKCLS2IYMfHe/MEaZOBtF1UxvXx8btmZ5\n6rnNPPrrjezeE5b7vmX6YZx98jGcOrODlmbtDC0i6dLwk7Z9fXk2bs+yZtNu1mzczZrNu1i3uZN9\nPWELlgnjW5k7+1jOPnkKR0/SOLyIpFdDBX5fPs/mHXtYs3E3qzftYu2m3azdvJt9ueL+Wk2ZDFM7\n2jhucjuzph3G262D1pYk/B0MEZH6aojA7+zKcdO9K3lx/U669/XuP5/JwJQj2jju6IlMP7qd4ya3\nc2zHBMa0KuBFRMo1ROCv2bSL37y8g8Pax3LK73Rw3OR23nT0RI49agJjFe4iIrE0ROAXzDllCuef\ncVy9qyEi0pC0VEVEJCUU+CIiKaHAFxFJCQW+iEhKKPBFRFJCgS8ikhIKfBGRlFDgi4ikhAJfRCQl\nFPgiIimhwBcRSQkFvohISijwRURSQoEvIpISCnwRkZRQ4IuIpIQCX0QkJRT4IiIpocAXEUkJBb6I\nSEoo8EVEUkKBLyKSEgp8EZGUUOCLiKSEAl9EJCUU+CIiKaHAFxFJCQW+iEhKKPBFRFJCgS8ikhIK\nfBGRlGiJU8jMzgUWES4QS9z92rLbJwJ3ANOAZuA6d7+1slUVEZGRGLKHb2ZNwGJgLnACMN/MZpUV\n+xyw0t1PBuYA15lZrIuJiIjURpwhndnAKndf6+45YCkwr6xMHmiPfm4Htrt7T+WqKSIiIxUn8KcA\n60qO10fnSi0G3mpmrwK/Ar5QmeqJiEilVGrYZS6wwt3fa2bHAw+Y2Ynu3jnYnTo62ge7eb9Dd3QB\n0NY2NvZ9Gs1ofV3DobYoUlsUqS1GLk7gbyBMxhZMjc6VuhT4vwDu/pKZrQZmAb8Y7IG3bt0dq5I7\nd+4BIJvdG/s+jaSjo31Uvq7hUFsUqS2K1BZFI7nwxQn8p4EZZjYd2AhcAMwvK7MWeD/wmJkdBcwE\nXh52rUREpOKGHMN3915gIXA/sBJY6u7Pm9kCM7ssKnYNcIaZPQs8AFzh7juqVWkRETl4scbw3f0+\nwMrO3Vjy80bCOL6IiCSUvmkrIpISCnwRkZRQ4IuIpIQCX0QkJRT4IiIp0RiBn693BUREGl9DBH7X\nvl4Axo1prnNNREQaV0MEfmdXDoAJ41vrXBMRkcalwBcRSYmGCPxsFPhtCnwRkWFriMBXD19EZOQU\n+CIiKdEQgZ/tytHclNEqHRGREWiIwO/sytE2roVMJlPvqoiINKyGCPxsd48mbEVERijxgd+Xz5Pt\nzmn8XkRkhBIf+Hu6e8jnNWErIjJSiQ98rcEXEamMxAe+lmSKiFSGAl9EJCUU+CIiKZH4wN8/hj9O\ngS8iMhKJD/zO7kIPv6XONRERaWyJD/xsVw+gVToiIiOV+MDXGL6ISGUo8EVEUiLxgZ/tyjF2TDMt\nzYmvqohIoiU+RTu7c0zQCh0RkRFLfuB3aeM0EZFKSHTg53p62Zfr05JMEZEKSHTgd2pJpohIxSQ8\n8LVTpohIpTRE4GvSVkRk5BId+FmtwRcRqZhEB35xHx0FvojISMVa/mJm5wKLCBeIJe5+bT9lzgG+\nDrQCW919zkgrp792JSJSOUP28M2sCVgMzAVOAOab2ayyMocCNwDnu/vbgE9UonLaVkFEpHLiDOnM\nBla5+1p3zwFLgXllZS4Elrn7BgB331aJyhUDX+vwRURGKk6STgHWlRyvJ1wESs0EWs3sQWACcL27\n3z7SyhW2RlYPX0Rk5Co1adsCnAqcB5wLXGVmM0b6oJ1dOZoyGcaPVQ9fRGSk4iTpBmBayfHU6Fyp\n9cA2d+8Gus3sYeAk4MXBHrijo33QJ+7a10t7WytHHjkxRjUb21BtkSZqiyK1RZHaYuTiBP7TwAwz\nmw5sBC4A5peV+RHwTTNrBsYCpwNfG+qBt27dPejtOzv3MmF865DlGl1HR/uof41xqS2K1BZFaoui\nkVz4hhzScfdeYCFwP7ASWOruz5vZAjO7LCrzArAceBZ4ErjJ3Z8bdq2AvnyebLd2yhQRqZRYg+Pu\nfh9gZeduLDv+J+CfKlWx7r095POasBURqZTEftO2uHGaJmxFRCohwYGvJZkiIpWU4MDXt2xFRCop\nsYGvfXRERCorsYGvvfBFRCor+YGvHr6ISEUkN/C7NaQjIlJJiQ18/bUrEZHKaoDA1zp8EZFKSGzg\nd3b1MKa1idaW5npXRURkVEhw4GsfHRGRSkpu4HfntCRTRKSCEhn4Pb197N3XqxU6IiIVlMjA1xp8\nEZHKU+CLiKREIgM/q62RRUQqLpGBr310REQqL5GBn+0Oe+Fr0lZEpHISGfgawxcRqTwFvohISijw\nRURSIpGBr792JSJSeYkM/M6uHJkMHDJOyzJFRColsYHfNq6Vpkym3lURERk1Ehn42a6chnNERCos\ncYGfz+fJdvcwQcM5IiIVlbjA79rbS29fXj18EZEKS1zgZ7u1JFNEpBoSF/hagy8iUh2JC3ytwRcR\nqY7EBb56+CIi1aHAFxFJieQGvpZliohUVOICP9ulvfBFRKohcYHfqWWZIiJVkbzA1yodEZGqiDVQ\nbmbnAosIF4gl7n7tAOVOAx4HPuXuPxhOhbJdOVpbmhjb2jycu4uIyACG7OGbWROwGJgLnADMN7NZ\nA5T7KrB8JBXq7MppOEdEpAriDOnMBla5+1p3zwFLgXn9lPs88O/AlpFUKNsdtkYWEZHKihP4U4B1\nJcfro3P7mdkxwEfc/dvAsDex7+nto2tvLxPGa0mmiEilVWrSdhFwZcnxsEI/2x2WZGpIR0Sk8uJ0\npTcA00qOp0bnSr0DWGpmGeAI4Dwzy7n7PYM9cEdH+xuOu3p3AXDEpLYDbhvt0vZ6B6O2KFJbFKkt\nRi5O4D8NzDCz6cBG4AJgfmkBd39z4WczuwW4d6iwB9i6dfcbjte9uhOAZvIH3DaadXS0p+r1DkZt\nUaS2KFJbFI3kwjfkkI679wILgfuBlcBSd3/ezBaY2WX93CU/3MrsX4OvSVsRkYqLNTvq7vcBVnbu\nxgHK/uFwK6ON00REqidR37TNKvBFRKomUYGvfXRERKonUYFf/GtXWocvIlJpiQr8zi6twxcRqZaE\nBX6ODFqlIyJSDYkK/GxXjkPGtdDUNOzdGUREZACJCvzOrpz2wRcRqZLEBH4+nw+Br+EcEZGqSEzg\nd+/rpbcvrwlbEZEqSUzgF790pSWZIiLVkJzAj7ZG1hi+iEh1JCbwtY+OiEh1KfBFRFJCgS8ikhKJ\nCfziPjoKfBGRakhM4O/v4WsdvohIVSQn8LU1sohIVSUn8LU1sohIVSUm8LNdOVqaM4xtba53VURE\nRqXEBH5h47RMRjtliohUQ2ICP9vVo/F7EZEqSkTg9/b1sWdvj1boiIhUUSICv7CPjnr4IiLVk4zA\n15euRESqLhGBr20VRESqT4EvIpISiQr8tnH60pWISLUkIvCzXZq0FRGptkQEfqcmbUVEqi4RgZ/V\nxmkiIlWXiMDXpK2ISPUlIvCz2ilTRKTqEhH4nV05xo9tobkpEdURERmVEpGwnV05Jqh3LyJSVXUP\n/Hw+T6d2yhQRqbq6B/6+XB89vX20aadMEZGqijWOYmbnAosIF4gl7n5t2e0XAldGh7uBP3H3X8d5\nbK3QERGpjSF7+GbWBCwG5gInAPPNbFZZsZeBs9z9JOAa4Oa4FdCXrkREaiNOD382sMrd1wKY2VJg\nHvBCoYC7P1lS/klgStwKdOpLVyIiNRFnDH8KsK7keD2DB/pngJ/ErUBWQzoiIjVR0bWQZjYHuBR4\nd5zyHR3tZFZtA2Dyke10dLRXsjoNJc2vvZzaokhtUaS2GLk4gb8BmFZyPDU69wZmdiJwE3Cuu78W\n58m3bt3Npi27Aejr6WHr1t1x7jbqdHS0p/a1l1NbFKktitQWRSO58MUJ/KeBGWY2HdgIXADMLy1g\nZtOAZcBF7v7SwVSgU1sji4jUxJBj+O7eCywE7gdWAkvd/XkzW2Bml0XFrgImAd8ysxVm9vO4Fdi/\nLFPr8EVEqirWGL673wdY2bkbS37+LPDZ4VSgsDWylmWKiFRX3b9p29mVo7kpw7gxzfWuiojIqJaI\nwG8b30omk6l3VURERrW6B362K6cJWxGRGqhr4Pf15dnT3cOEcdoaWUSk2uoa+Hv29pBHE7YiIrVQ\n18DXTpkiIrWjwBcRSQkFvohIStQ18LPaC19EpGYS0cPXnzcUEam+RAT+hPFalikiUm2JGNLRGL6I\nSPUlpIevwBcRqbb69vC7w174mrQVEam+uvfwx41ppqW57lv6iIiMenUPfA3niIjURt0nbTWcIyJS\nG3UL/L25Xvb19KmHLyJSI3UL/N3ZfQC0aWtkEZGaqF/g7wmBrx6+iEht1C3wd2UV+CIitVT3Hr4m\nbUVEaqPuY/jq4YuI1EYde/jaVkFEpJbqPqSjwBcRqY26T9pqDF9EpDbq38PXHz8REamJuk7aNmUy\njB/bXK8qiIikSl17+G3jW8hkMvWqgohIqtRxDF87ZYqI1FLdAj/btU8TtiIiNVS3wO/La8JWRKSW\n6rofvoZ0RERqR4EvIpISdQ38tvHaC19EpFZiJa6ZnQssIlwglrj7tf2UuR44D8gCn3b3Z4Z6XPXw\nRURqZ8gevpk1AYuBucAJwHwzm1VW5jzgeHf/HWAB8J04T67AFxGpnThDOrOBVe6+1t1zwFJgXlmZ\necBtAO7+FHComR011AO3aZWOiEjNxAn8KcC6kuP10bnBymzop8wB1MMXEamd+q7SOUSBLyJSK3Em\nbTcA00qOp0bnysscO0SZN7j3unnaRKdER0d7vauQGGqLIrVFkdpi5OL08J8GZpjZdDMbA1wA3FNW\n5h7gYgAzeyfwurtvrmhNRURkRIYMfHfvBRYC9wMrgaXu/ryZLTCzy6IyPwZWm9mLwI3A5VWss4iI\nDEMmn8/Xuw4iIlIDdZ20FRGR2lHgi4ikhAJfRCQlqr57WbX24WlEQ7WFmV0IXBkd7gb+xN1/Xdta\n1kac90VU7jTgceBT7v6DGlaxZmL+jpwDfB1oBba6+5yaVrJGYvyOTATuICwVbwauc/dba13PajOz\nJcD5wGZ3P3GAMgedm1Xt4VdzH55GE6ctgJeBs9z9JOAa4Oba1rI2YrZFodxXgeW1rWHtxPwdORS4\nATjf3d8GfKLmFa2BmO+LzwEr3f1kYA5wnZmNxm13byG0Q7+Gm5vVHtKp2j48DWjItnD3J919Z3T4\nJDG2p2hQcd4XAJ8H/h3YUsvK1VictrgQWObuGwDcfVuN61grcdoiDxS+gdUObHf3nhrWsSbc/VHg\ntUGKDCs3qx34VduHpwHFaYtSnwF+UtUa1c+QbWFmxwAfcfdvA6P5W9lx3hczgUlm9qCZPW1mF9Ws\ndrUVpy0WA281s1eBXwFfqFHdkmZYualJ2wQysznApRTH89NoEW98/aM59IfSApxKGK89F7jKzGbU\nt0p1MxdY4e7HAKcAN5jZhDrXqWFUO/Crsg9Pg4rTFpjZicBNwIfdfbCPdI0sTlu8A1hqZquBjxN+\nsT9co/rVUpy2WA8sd/dud98OPAycVKP61VKctrgU+AGAu78ErAYOmP9JgWHlZrUnO/bvwwNsJOzD\nM7+szD2EiZi7Rvk+PEO2hZlNA5YBF0Vv5tFqyLZw9zcXfjazW4B73b18D6fRIM7vyI+Ab5pZMzAW\nOB34Wk1rWRtx2mIt8H7gsWjMeiZhscNolGHgT7bDys2q9vC1D09RnLYArgImAd8ysxVm9vM6Vbeq\nYrZFqVG7/0fM35EXCCuVniVM5t/k7s/Vq87VEvN9cQ1whpk9CzwAXOHuO+pT4+oxszsJy5Fnmtkr\nZnZpJXJTe+mIiKSEJm1FRFJCgS8ikhIKfBGRlFDgi4ikhAJfRCQlFPgiIimhwBcRSQkFvohISvx/\nE2bsie0eY5UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f72ac2daba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "@interact(log_of_n_tree=(0,3,0.1))\n",
    "def roc(log_of_n_tree=1):\n",
    "    rslt=RFC(n_estimators=int(10**log_of_n_tree),n_jobs=-1).fit(data_train,target_train).predict_proba(data_test)\n",
    "    fp_ratio, tp_ratio, thresholds = metrics.roc_curve(target_test,rslt[:,1])\n",
    "    auc=metrics.roc_auc_score(target_test,rslt[:,1])\n",
    "    plt.plot(fp_ratio,tp_ratio)\n",
    "    plt.title(\"ROC Curve:n_trees={0}; auc={1:.3}\".format(int(10**log_of_n_tree),auc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# スコアリング関数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import cross_validation as cv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "これで出てくる、scoringの話です"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "cross validationで使えるスコアリング方法はmetrics.scorer.SCORERSで定義されています"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['accuracy', 'precision', 'f1_macro', 'recall_macro', 'recall', 'recall_micro', 'f1_weighted', 'recall_samples', 'f1', 'precision_weighted', 'average_precision', 'roc_auc', 'f1_samples', 'median_absolute_error', 'mean_squared_error', 'precision_macro', 'mean_absolute_error', 'adjusted_rand_score', 'precision_samples', 'precision_micro', 'recall_weighted', 'r2', 'log_loss', 'f1_micro']\n"
     ]
    }
   ],
   "source": [
    "print(list(metrics.scorer.SCORERS.keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "- 用意されているSCORER以外にもスコアリング関数を定義することが可能です。\n",
    "- 下記形式で予測結果に対してスカラー量を算出できる関数を作成すれば、make_scorerでSCORERとして使えます。\n",
    "> score_func(target, predict_proba)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "custom_scorer=metrics.scorer.make_scorer(metrics.roc_auc_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "こんな感じ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.96585648,  0.94733796,  0.90625   ,  0.97169811,  0.93274498])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_method=cv.StratifiedKFold(target_train,shuffle=True,n_folds=5)\n",
    "cv.cross_val_score(clf,data_train,target_train,cv=cv_method,scoring=custom_scorer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "- 実際には、scoring=\"roc_auc\"で動作するSCORERはほぼ同じ方法で実装されています"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "make_scorer(roc_auc_score, needs_threshold=True)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.scorer.SCORERS[\"roc_auc\"]"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}