From 447b1e2b5f750b0bb60d1cf6e3cf64ec4cd7a0b1 Mon Sep 17 00:00:00 2001 From: MaoYZ Date: Tue, 17 Sep 2024 18:03:04 +0800 Subject: [PATCH] =?UTF-8?q?=E8=AF=84=E4=BB=B7=E5=88=86=E7=B1=BB=E7=BB=93?= =?UTF-8?q?=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...onfusion-Matrix-Precision-and-Recall.ipynb | 446 +++++++++++++ .../02-F1Score/F1Score.ipynb | 354 ++++++++++ .../precisionRecallTradeoff.ipynb | 629 ++++++++++++++++++ .../06-Precision-Recall-Curve.ipynb | 285 ++++++++ .../07-ROC-Curve/07-ROC-Curve.ipynb | 198 ++++++ .../07-ROC-Curve/playML/LinearRegression.py | 116 ++++ .../07-ROC-Curve/playML/LogisticRegression.py | 82 +++ .../07-ROC-Curve/playML/PCA.py | 71 ++ .../playML/SimpleLinearRegression.py | 47 ++ .../07-ROC-Curve/playML/__init__.py | 0 .../07-ROC-Curve/playML/kNN.py | 60 ++ .../07-ROC-Curve/playML/metrics.py | 114 ++++ .../07-ROC-Curve/playML/model_selection.py | 26 + .../07-ROC-Curve/playML/preprocessing.py | 30 + ...-Matrix-in-Multiclass-Classification.ipynb | 230 +++++++ 15 files changed, 2688 insertions(+) create mode 100644 machinelearning/classificationPerformanceMeasures/01-implementConfusionMatrixPrecisionAndRecall/Implement-Confusion-Matrix-Precision-and-Recall.ipynb create mode 100644 machinelearning/classificationPerformanceMeasures/02-F1Score/F1Score.ipynb create mode 100644 machinelearning/classificationPerformanceMeasures/03-PrecisionRecallTradeoff/precisionRecallTradeoff.ipynb create mode 100644 machinelearning/classificationPerformanceMeasures/04-precisionRecallCurve/06-Precision-Recall-Curve.ipynb create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/07-ROC-Curve.ipynb create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LinearRegression.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LogisticRegression.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/PCA.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/SimpleLinearRegression.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/__init__.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/kNN.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/metrics.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/model_selection.py create mode 100644 machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/preprocessing.py create mode 100644 machinelearning/classificationPerformanceMeasures/08-Confusion-Matrix-in-Multiclass-Classification/08-Confusion-Matrix-in-Multiclass-Classification.ipynb diff --git a/machinelearning/classificationPerformanceMeasures/01-implementConfusionMatrixPrecisionAndRecall/Implement-Confusion-Matrix-Precision-and-Recall.ipynb b/machinelearning/classificationPerformanceMeasures/01-implementConfusionMatrixPrecisionAndRecall/Implement-Confusion-Matrix-Precision-and-Recall.ipynb new file mode 100644 index 0000000..ecc2061 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/01-implementConfusionMatrixPrecisionAndRecall/Implement-Confusion-Matrix-Precision-and-Recall.ipynb @@ -0,0 +1,446 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 实现混淆矩阵，精准率和召回率" + ] + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.378021Z", + "start_time": "2024-09-16T01:48:07.782453Z" + } + }, + "source": [ + "import numpy as np\n", + "from sklearn import datasets" + ], + "outputs": [], + "execution_count": 1 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.394394Z", + "start_time": "2024-09-16T01:48:08.380169Z" + } + }, + "source": [ + "digits = datasets.load_digits()\n", + "X = digits.data\n", + "y = digits.target.copy()\n", + "\n", + "y[digits.target==9] = 1\n", + "y[digits.target!=9] = 0" + ], + "outputs": [], + "execution_count": 2 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.442789Z", + "start_time": "2024-09-16T01:48:08.395279Z" + } + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)" + ], + "outputs": [], + "execution_count": 3 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.508745Z", + "start_time": "2024-09-16T01:48:08.443762Z" + } + }, + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "log_reg = LogisticRegression()\n", + "log_reg.fit(X_train, y_train)\n", + "log_reg.score(X_test, y_test)" + ], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/myenv3.10/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):\n", + "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n", + "\n", + "Increase the number of iterations (max_iter) or scale the data as shown in:\n", + " https://scikit-learn.org/stable/modules/preprocessing.html\n", + "Please also refer to the documentation for alternative solver options:\n", + " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", + " n_iter_i = _check_optimize_result(\n" + ] + }, + { + "data": { + "text/plain": [ + "0.9755555555555555" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 4 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.514157Z", + "start_time": "2024-09-16T01:48:08.511430Z" + } + }, + "source": [ + "y_log_predict = log_reg.predict(X_test)" + ], + "outputs": [], + "execution_count": 5 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.518999Z", + "start_time": "2024-09-16T01:48:08.515455Z" + } + }, + "source": [ + "def TN(y_true, y_predict):\n", + " assert len(y_true) == len(y_predict)\n", + " return np.sum((y_true == 0) & (y_predict == 0))\n", + "\n", + "TN(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(403)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 6 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.524849Z", + "start_time": "2024-09-16T01:48:08.520092Z" + } + }, + "source": [ + "def FP(y_true, y_predict):\n", + " assert len(y_true) == len(y_predict)\n", + " return np.sum((y_true == 0) & (y_predict == 1))\n", + "\n", + "FP(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(2)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 7 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.531567Z", + "start_time": "2024-09-16T01:48:08.527127Z" + } + }, + "source": [ + "def FN(y_true, y_predict):\n", + " assert len(y_true) == len(y_predict)\n", + " return np.sum((y_true == 1) & (y_predict == 0))\n", + "\n", + "FN(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(9)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 8 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.537621Z", + "start_time": "2024-09-16T01:48:08.532962Z" + } + }, + "source": [ + "def TP(y_true, y_predict):\n", + " assert len(y_true) == len(y_predict)\n", + " return np.sum((y_true == 1) & (y_predict == 1))\n", + "\n", + "TP(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(36)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 9 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.542845Z", + "start_time": "2024-09-16T01:48:08.538908Z" + } + }, + "source": [ + "def confusion_matrix(y_true, y_predict):\n", + " return np.array([\n", + " [TN(y_true, y_predict), FP(y_true, y_predict)],\n", + " [FN(y_true, y_predict), TP(y_true, y_predict)]\n", + " ])\n", + "\n", + "confusion_matrix(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[403, 2],\n", + " [ 9, 36]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 10 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.548048Z", + "start_time": "2024-09-16T01:48:08.543903Z" + } + }, + "source": [ + "def precision_score(y_true, y_predict):\n", + " tp = TP(y_true, y_predict)\n", + " fp = FP(y_true, y_predict)\n", + " try:\n", + " return tp / (tp + fp)\n", + " except:\n", + " return 0.0\n", + " \n", + "precision_score(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.9473684210526315)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 11 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.553272Z", + "start_time": "2024-09-16T01:48:08.549306Z" + } + }, + "source": [ + "def recall_score(y_true, y_predict):\n", + " tp = TP(y_true, y_predict)\n", + " fn = FN(y_true, y_predict)\n", + " try:\n", + " return tp / (tp + fn)\n", + " except:\n", + " return 0.0\n", + " \n", + "recall_score(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### scikit-learn中的混淆矩阵，精准率和召回率" + ] + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.558956Z", + "start_time": "2024-09-16T01:48:08.554038Z" + } + }, + "source": [ + "from sklearn.metrics import confusion_matrix\n", + "\n", + "confusion_matrix(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[403, 2],\n", + " [ 9, 36]])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 13 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.567855Z", + "start_time": "2024-09-16T01:48:08.562536Z" + } + }, + "source": [ + "from sklearn.metrics import precision_score\n", + "\n", + "precision_score(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.9473684210526315)" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 14 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:48:08.573824Z", + "start_time": "2024-09-16T01:48:08.568712Z" + } + }, + "source": [ + "from sklearn.metrics import recall_score\n", + "\n", + "recall_score(y_test, y_log_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 15 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/machinelearning/classificationPerformanceMeasures/02-F1Score/F1Score.ipynb b/machinelearning/classificationPerformanceMeasures/02-F1Score/F1Score.ipynb new file mode 100644 index 0000000..d084b22 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/02-F1Score/F1Score.ipynb @@ -0,0 +1,354 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## F1 Score" + ] + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.255143Z", + "start_time": "2024-09-16T01:49:01.213811Z" + } + }, + "source": [ + "import numpy as np" + ], + "outputs": [], + "execution_count": 1 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.259777Z", + "start_time": "2024-09-16T01:49:01.256712Z" + } + }, + "source": [ + "def f1_score(precision, recall):\n", + " try:\n", + " return 2 * precision * recall / (precision + recall)\n", + " except:\n", + " return 0.0" + ], + "outputs": [], + "execution_count": 2 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.265857Z", + "start_time": "2024-09-16T01:49:01.261297Z" + } + }, + "source": [ + "precision = 0.5\n", + "recall = 0.5\n", + "f1_score(precision, recall)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "0.5" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 3 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.271443Z", + "start_time": "2024-09-16T01:49:01.267247Z" + } + }, + "source": [ + "precision = 0.1\n", + "recall = 0.9\n", + "f1_score(precision, recall)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "0.18000000000000002" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 4 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.277461Z", + "start_time": "2024-09-16T01:49:01.274094Z" + } + }, + "source": [ + "precision = 0.0\n", + "recall = 1.0\n", + "f1_score(precision, recall)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 5 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.737431Z", + "start_time": "2024-09-16T01:49:01.278368Z" + } + }, + "source": [ + "from sklearn import datasets\n", + "\n", + "digits = datasets.load_digits()\n", + "X = digits.data\n", + "y = digits.target.copy()\n", + "\n", + "y[digits.target==9] = 1\n", + "y[digits.target!=9] = 0" + ], + "outputs": [], + "execution_count": 6 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.778433Z", + "start_time": "2024-09-16T01:49:01.738455Z" + } + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)" + ], + "outputs": [], + "execution_count": 7 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.838414Z", + "start_time": "2024-09-16T01:49:01.779238Z" + } + }, + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "log_reg = LogisticRegression()\n", + "log_reg.fit(X_train, y_train)\n", + "# 准确率\n", + "log_reg.score(X_test, y_test)" + ], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/myenv3.10/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):\n", + "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n", + "\n", + "Increase the number of iterations (max_iter) or scale the data as shown in:\n", + " https://scikit-learn.org/stable/modules/preprocessing.html\n", + "Please also refer to the documentation for alternative solver options:\n", + " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", + " n_iter_i = _check_optimize_result(\n" + ] + }, + { + "data": { + "text/plain": [ + "0.9755555555555555" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 8 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.842496Z", + "start_time": "2024-09-16T01:49:01.839572Z" + } + }, + "source": [ + "y_predict = log_reg.predict(X_test)" + ], + "outputs": [], + "execution_count": 9 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.848941Z", + "start_time": "2024-09-16T01:49:01.843599Z" + } + }, + "source": [ + "from sklearn.metrics import confusion_matrix\n", + "\n", + "confusion_matrix(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[403, 2],\n", + " [ 9, 36]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 10 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.857132Z", + "start_time": "2024-09-16T01:49:01.850617Z" + } + }, + "source": [ + "from sklearn.metrics import precision_score\n", + "\n", + "precision_score(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.9473684210526315)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 11 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.864113Z", + "start_time": "2024-09-16T01:49:01.858452Z" + } + }, + "source": [ + "from sklearn.metrics import recall_score\n", + "\n", + "recall_score(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 12 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T01:49:01.870655Z", + "start_time": "2024-09-16T01:49:01.865342Z" + } + }, + "source": [ + "from sklearn.metrics import f1_score\n", + "\n", + "f1_score(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8674698795180723)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 13 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/machinelearning/classificationPerformanceMeasures/03-PrecisionRecallTradeoff/precisionRecallTradeoff.ipynb b/machinelearning/classificationPerformanceMeasures/03-PrecisionRecallTradeoff/precisionRecallTradeoff.ipynb new file mode 100644 index 0000000..010eb17 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/03-PrecisionRecallTradeoff/precisionRecallTradeoff.ipynb @@ -0,0 +1,629 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 精准度和召回率的平衡" + ] + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.036128Z", + "start_time": "2024-09-16T09:44:47.732636Z" + } + }, + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ], + "outputs": [], + "execution_count": 1 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.516299Z", + "start_time": "2024-09-16T09:44:48.037763Z" + } + }, + "source": [ + "from sklearn import datasets\n", + "\n", + "digits = datasets.load_digits()\n", + "X = digits.data\n", + "y = digits.target.copy()\n", + "\n", + "y[digits.target==9] = 1\n", + "y[digits.target!=9] = 0" + ], + "outputs": [], + "execution_count": 2 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.557796Z", + "start_time": "2024-09-16T09:44:48.517192Z" + } + }, + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)" + ], + "outputs": [], + "execution_count": 3 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.621060Z", + "start_time": "2024-09-16T09:44:48.558958Z" + } + }, + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "log_reg = LogisticRegression()\n", + "log_reg.fit(X_train, y_train)\n", + "y_predict = log_reg.predict(X_test)" + ], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/anaconda3/envs/myenv3.10/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):\n", + "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n", + "\n", + "Increase the number of iterations (max_iter) or scale the data as shown in:\n", + " https://scikit-learn.org/stable/modules/preprocessing.html\n", + "Please also refer to the documentation for alternative solver options:\n", + " https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n", + " n_iter_i = _check_optimize_result(\n" + ] + } + ], + "execution_count": 4 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.630035Z", + "start_time": "2024-09-16T09:44:48.623437Z" + } + }, + "source": [ + "from sklearn.metrics import f1_score\n", + "\n", + "f1_score(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8674698795180723)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 5 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.635696Z", + "start_time": "2024-09-16T09:44:48.630895Z" + } + }, + "source": [ + "from sklearn.metrics import confusion_matrix\n", + "\n", + "confusion_matrix(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[403, 2],\n", + " [ 9, 36]])" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 6 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.641379Z", + "start_time": "2024-09-16T09:44:48.636644Z" + } + }, + "source": [ + "from sklearn.metrics import precision_score\n", + "\n", + "precision_score(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.9473684210526315)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 7 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.647216Z", + "start_time": "2024-09-16T09:44:48.642367Z" + } + }, + "source": [ + "from sklearn.metrics import recall_score\n", + "\n", + "recall_score(y_test, y_predict)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 8 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.651854Z", + "start_time": "2024-09-16T09:44:48.648306Z" + } + }, + "source": [ + "log_reg.decision_function(X_test)[:10]" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([-21.39632803, -32.89836755, -16.42503064, -79.82740073,\n", + " -48.03006253, -24.18016394, -44.61773888, -24.2382381 ,\n", + " -1.1442306 , -19.00687291])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 9 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.655747Z", + "start_time": "2024-09-16T09:44:48.652693Z" + } + }, + "source": [ + "log_reg.predict(X_test)[:10]" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 10 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.661423Z", + "start_time": "2024-09-16T09:44:48.656830Z" + } + }, + "source": [ + "decision_scores = log_reg.decision_function(X_test)\n", + "print(decision_scores) # 每个样本的 score 值 " + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[-21.39632803 -32.89836755 -16.42503064 -79.82740073 -48.03006253\n", + " -24.18016394 -44.61773888 -24.2382381 -1.1442306 -19.00687291\n", + " -65.82579222 -50.97384945 -30.91383804 -45.95001961 -37.36079287\n", + " -29.52191392 -36.92525489 -82.81936974 -37.64655242 -9.87455985\n", + " -9.27181223 -85.26072092 -16.75200555 -45.34187288 -5.02565958\n", + " -48.30720246 -11.65686706 -37.35115796 -25.08294084 -13.59519777\n", + " -16.59094522 -28.7868136 -34.37187546 -28.52250796 -8.12118919\n", + " -4.61236199 -21.9213878 -21.88101143 -31.15032516 -23.37622916\n", + " -26.90704067 -62.23845924 -37.68292314 -66.36540349 -20.10762829\n", + " -16.67567208 -18.17120489 -21.54706516 -28.96499959 -19.61221275\n", + " 2.42106669 7.72579943 -34.86440796 -42.71126562 -25.63790717\n", + " -34.75654341 -7.59704356 -49.51935699 -51.52679169 19.66635111\n", + " -10.10862009 -32.01145468 -11.49933661 -1.44233748 -48.69944223\n", + " -43.86134828 -24.84126264 -19.60577522 -36.64996545 -3.52583948\n", + " -4.45221101 -19.2151373 -20.35656157 -40.89711366 -11.85824617\n", + " -32.75336536 -35.76186227 -28.58890488 -55.41752828 -18.83061949\n", + " 4.57091465 -16.463646 -76.78227343 -58.24102588 -30.23791246\n", + " -29.423417 -33.41427822 -8.41635365 -47.91948007 -65.49892246\n", + " -16.91630447 -22.17290895 -11.28295144 -18.67186697 -69.22923712\n", + " -46.39154589 -39.45401269 -35.9335525 -17.7303169 -62.96898278\n", + " -16.86305036 -55.15268001 -28.77455676 -68.48799082 -68.86724305\n", + " -6.50066996 -25.5166632 -38.3237795 -27.4657971 -15.54534665\n", + " -27.47967996 -20.34240725 12.07585494 -23.09377024 -35.9749607\n", + " -29.87345938 -68.96188671 -27.31294693 -54.25065148 -24.62891432\n", + " -11.84731319 -47.36810768 -2.7552612 -59.69176582 -30.99290537\n", + " -8.99688404 -70.8437001 -56.98391867 -20.07087563 -21.50466085\n", + " -68.2885106 -18.91402081 -38.59546834 -57.36443298 -0.91127195\n", + " -22.5230999 -22.66313156 -29.00758695 -32.76608118 -20.44146722\n", + " -11.35208442 4.63883739 6.26793819 1.48978716 -7.63611926\n", + " -39.24568643 12.15963601 -74.54758286 -75.08976825 -49.97540869\n", + " -11.63726915 -47.61549493 -75.41844924 -29.90338777 -63.94066399\n", + " -7.26363237 -6.64195494 -18.2155006 -32.47359246 -17.94096338\n", + " -43.32139635 -32.70198994 -34.305205 -72.74642491 -15.19453349\n", + " 11.47711832 -56.41444251 -6.03679672 -48.38174285 -16.44342301\n", + " -2.13532057 -11.85856169 -33.2615532 -51.35612964 -10.3854679\n", + " -17.18931257 -5.24186043 -25.20231775 -15.70408888 3.5512389\n", + " -45.02515953 -12.57784626 -25.37484849 -16.5760627 -22.16247652\n", + " -82.49793825 -5.87782175 -20.26924964 -20.46798451 -26.81797463\n", + " -25.98205563 -40.46368473 -38.00370623 -26.96407202 -23.7556394\n", + " -20.14896398 -9.68706827 -19.68267804 -42.50630152 -44.1474587\n", + " -15.65592717 -64.03595061 -24.55492892 -56.30748473 -13.0153844\n", + " -29.66152689 3.89274429 -44.3372052 -7.91496387 1.14288279\n", + " -2.82200976 -11.9298259 7.50919056 -7.17677196 -46.38967231\n", + " -48.65790438 -4.59703312 -19.04918093 -24.07425902 -48.76420724\n", + " -15.02118533 -24.92911043 -16.69731748 -18.68076058 -15.70026039\n", + " -16.86891478 -38.53342161 -31.09474143 -9.38184941 -71.45269781\n", + " -22.77486554 -14.43346467 -23.07982404 -34.32201506 -0.88829297\n", + " -32.74800984 -11.22610884 -18.67835381 -8.21183045 -45.4430361\n", + " -22.3048531 -62.4046439 -46.77074565 -65.15482912 -33.22979898\n", + " -23.47223796 -28.50164243 -64.79125993 1.45396815 -4.09350604\n", + " -25.6560368 -22.31931787 -54.69329449 -16.34265394 -12.07699182\n", + " -35.28803117 -5.74561325 -13.50697395 -72.29504081 -6.16616975\n", + " -1.16745264 -35.57074284 -24.16464549 -68.32001285 14.76664432\n", + " -63.07044866 9.9114665 -24.14614137 -32.45946469 -14.39403155\n", + " -85.73589062 -12.78449394 8.99700957 -16.51275081 -36.68112125\n", + " -16.5142318 -19.35489432 -32.59144615 -5.64199251 7.68831562\n", + " 9.39582472 5.86052677 -35.64855143 -12.98554895 -54.42604517\n", + " -41.10680277 5.62961814 -79.49290143 -15.82339505 -19.23222952\n", + " -10.86523966 -42.52431569 -19.82259121 -15.70234888 -17.99450127\n", + " -18.02770039 -6.75709263 -20.78875363 -16.58248998 -70.42953085\n", + " -9.20996399 -31.69729569 -19.68492641 -21.96674718 -24.76965523\n", + " -16.38407636 -13.36979769 -22.93073434 11.05839158 -15.38201515\n", + " -32.94035842 -13.74990956 -50.36070675 -20.46520927 -56.27361211\n", + " -28.68660012 -21.86436781 -30.41316875 -69.26213003 -59.34758793\n", + " 14.34666127 8.58023128 -25.67055443 2.74201107 4.9349716\n", + " -19.67145243 -58.83290391 -10.0106847 -28.80840929 -27.20569285\n", + " 6.29303171 -80.47836457 -34.4594727 -50.2950448 -35.95426284\n", + " -48.638095 -17.99381876 -62.34571387 -3.09533027 -25.26144012\n", + " -64.10550641 -9.62433703 -21.75093177 19.90895049 -18.75278074\n", + " -4.47036571 -13.15064794 -21.64150504 -43.10209091 -52.11204298\n", + " -28.5267845 -14.55989702 -2.47300226 -6.12313765 3.69927708\n", + " -15.00324649 -40.85549694 -26.65311131 14.10932393 -17.69128675\n", + " 15.19041957 -33.09240839 5.26736074 -14.27034185 -53.58823639\n", + " -50.03864289 -30.66777869 -38.04959657 -23.29430945 -24.69534374\n", + " -13.56655803 -22.61684297 -27.22295453 -19.64547188 -28.17465092\n", + " -19.93511498 -29.83095565 -11.29200116 -17.24466233 -24.03020454\n", + " -24.35711168 10.3927233 -17.22134301 -38.02469802 -16.08880146\n", + " -37.58627833 -16.33661862 -69.12719912 -33.68695612 -43.62833897\n", + " -26.59250139 -10.31848973 -66.36421923 -31.89660769 -45.56274349\n", + " -14.58211052 -36.12438586 -14.95239118 -70.02153238 -11.35930794\n", + " -40.86598947 -32.66642135 -19.76342546 -27.57777649 -15.73319443\n", + " -31.58141356 -8.5131452 -21.37778539 -34.07581735 -11.68164353\n", + " -36.4319518 -34.77814158 -22.22229751 4.77640054 -21.31951689\n", + " -4.45913305 -20.83062634 -32.25695828 -41.12662624 -25.08183831\n", + " -19.7625071 -47.87267406 -30.91359757 -45.56398272 -71.52302451\n", + " -6.25666324 -32.55674222 2.28031179 11.94364146 7.115445\n", + " -31.37474547 -63.96093128 -23.79015443 -5.7376593 -32.42331007\n", + " -24.72419264 -67.70498722 -32.82676878 -33.60753273 -31.53836834\n", + " -51.97739661 -22.54373289 -7.74506607 -17.29730481 -25.78028594\n", + " -32.38263295 -29.49540535 -66.43502146 -45.69850615 -16.05293553]\n" + ] + } + ], + "execution_count": 11 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.665244Z", + "start_time": "2024-09-16T09:44:48.662351Z" + } + }, + "source": [ + "np.min(decision_scores)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(-85.73589062374965)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 12 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.670779Z", + "start_time": "2024-09-16T09:44:48.666938Z" + } + }, + "source": [ + "np.max(decision_scores)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(19.908950489969612)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 13 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.676526Z", + "start_time": "2024-09-16T09:44:48.674396Z" + } + }, + "source": [ + "# 使用新的预测值， 默认这个decision_scores =0 ，>=5分类为1，<5分类为0 \n", + "y_predict_2 = np.array(decision_scores >= 5, dtype='int')" + ], + "outputs": [], + "execution_count": 14 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.681893Z", + "start_time": "2024-09-16T09:44:48.677709Z" + } + }, + "source": [ + "confusion_matrix(y_test, y_predict_2)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[404, 1],\n", + " [ 21, 24]])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 15 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.687884Z", + "start_time": "2024-09-16T09:44:48.683104Z" + } + }, + "source": [ + "precision_score(y_test, y_predict_2)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.96)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 16 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.693952Z", + "start_time": "2024-09-16T09:44:48.688880Z" + } + }, + "source": [ + "recall_score(y_test, y_predict_2)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.5333333333333333)" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 17 + }, + { + "cell_type": "code", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.697095Z", + "start_time": "2024-09-16T09:44:48.694898Z" + } + }, + "source": [ + "# >=-5分类为1 小于-5 分类为0\n", + "y_predict_3 = np.array(decision_scores >= -5, dtype='int')" + ], + "outputs": [], + "execution_count": 18 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.702119Z", + "start_time": "2024-09-16T09:44:48.698324Z" + } + }, + "source": [ + "confusion_matrix(y_test, y_predict_3)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "array([[390, 15],\n", + " [ 5, 40]])" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 19 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.707863Z", + "start_time": "2024-09-16T09:44:48.703023Z" + } + }, + "source": [ + "precision_score(y_test, y_predict_3)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.7272727272727273)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 20 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2024-09-16T09:44:48.713875Z", + "start_time": "2024-09-16T09:44:48.708779Z" + } + }, + "source": [ + "recall_score(y_test, y_predict_3)" + ], + "outputs": [ + { + "data": { + "text/plain": [ + "np.float64(0.8888888888888888)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 21 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/machinelearning/classificationPerformanceMeasures/04-precisionRecallCurve/06-Precision-Recall-Curve.ipynb b/machinelearning/classificationPerformanceMeasures/04-precisionRecallCurve/06-Precision-Recall-Curve.ipynb new file mode 100644 index 0000000..902d037 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/04-precisionRecallCurve/06-Precision-Recall-Curve.ipynb @@ -0,0 +1,285 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 精准度-召回率曲线" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "\n", + "digits = datasets.load_digits()\n", + "X = digits.data\n", + "y = digits.target.copy()\n", + "\n", + "y[digits.target==9] = 1\n", + "y[digits.target!=9] = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "log_reg = LogisticRegression()\n", + "log_reg.fit(X_train, y_train)\n", + "decision_scores = log_reg.decision_function(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.metrics import precision_score\n", + "from sklearn.metrics import recall_score\n", + "\n", + "precisions = []\n", + "recalls = []\n", + "thresholds = np.arange(np.min(decision_scores), np.max(decision_scores), 0.1)\n", + "for threshold in thresholds:\n", + " y_predict = np.array(decision_scores >= threshold, dtype='int')\n", + " precisions.append(precision_score(y_test, y_predict))\n", + " recalls.append(recall_score(y_test, y_predict))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8HNV99/HPb3elXd0syZZkfL8hG8zFGIyBBgcIEDCk\nISk0ISFcQoCSAiFN24SWpn3ypG0gJH1CGxJKCSE0SQkNAUxCaiDlbi42d4xtbGxky1f5oru00u6e\n549Z27KQrZW00mhnv+/XS6/ZnRnt/Mayvz46c+aMOecQEZFgCfldgIiIZJ/CXUQkgBTuIiIBpHAX\nEQkghbuISAAp3EVEAkjhLiISQAp3EZEAUriLiARQxK8DV1VVuenTp/t1eBGRnPTqq6/udM5V97ef\nb+E+ffp0VqxY4dfhRURykpnVZbKfumVERAJI4S4iEkAKdxGRAFK4i4gEkMJdRCSA+g13M7vHzHaY\n2TsH2W5m9q9mts7M3jKz47NfpoiIDEQmLfd7gXMPsX0xUJv+ugb48dDLEhGRoeh3nLtz7lkzm36I\nXS4A7nPe8/peMrMKM5vgnNuapRoPtP1dWPnQsHy0SE6LFMKJV0FRpd+V+GpLYwcPrNhEKjV6HyG6\nYPpYPjq73/uQhiQbNzFNAjb1eF+fXvehcDeza/Ba90ydOnVwR9u5Bp69bXDfKxJY6SArmwjzL/G3\nFJ89sGITP3hyLWZ+V3Jw1542KyfCPWPOubuAuwAWLFgwuP9Wj/q09yUi+3U2wS1ToWOP35X4rqM7\nSTQSYs0/Lva7FF9lY7TMZmBKj/eT0+tEZKQUlgHmhXyei3eniEY0EDAbfwJLgMvSo2ZOBpqGrb9d\nRPoWCkGsHDp2QzIBbvT2Nw+3eCJFtCDsdxm+67dbxsz+CzgdqDKzeuAfgAIA59ydwGPAecA6oB34\n4nAVKyKHUDwWlt/tfdUcBX++zO+KfBFPJNVyJ7PRMp/rZ7sDrstaRSIyOOf/C9SvgLrnYf3TXgs+\n7NvEr1m3aXc7zZ3dFIZDHF5Tih3kimk8oW4Z8HHKXxHJsllneF8vlXnhHm/2WvMBsKWxg0XffeqA\ndcWFYW48s5Y/mlXFMZPL9633+tzVLaNwFwmaWDroOvYEJty3N3cC8JWPHU51WZTtzXGWvb+T7/x+\nNQAnzxzLNR+dybzJFbR3JYgWqOWucBcJmqIKb/n8v0Dp+Ox//qwzYfpHsv+5h9DelQTg1NpqFs7w\n/sNKJGt5e3MTD7++mV+8vJGX1u9/+E9pVNGmPwGRoKma7bXe37w/+5+dSsAHL8CXlmb/sw+hNZ4A\noCS6v7slEg4xf2ol86dW8tWzZrN05Ta2NXfygyfXctKMYPzGMhQKd5GgGTcLbto4PJ/9wGWwY3XW\nP3ZbUydX/PQV2roSB6wPm/HtTx1Ne3p9SWHfkVVZUsjFC7273q9eNJNwaBTfnjpCFO4ikrlY+bDc\nKLVqWzOrt7VwxpxqKosL963/3zU7+Pqv3+LoSd51hOJo/xdKS9QlAyjcRWQgYhXQ2QiJ+IHrLTyk\nYZctnV7L/Obzj+TwmrJ96x96vZ5v/Pptnnh3O4WREGNiBYM+Rr5RuItI5orHQqIT/rHmwPWRIvjy\nC16X0CC0dHYDUNYrvD89fzKLj57Am5saqS6LEtOdpxlTuItI5o67BCzkXVjdq3mLd1fs7vWDCvdt\nTZ28Xe919ZTFPhxJsYIwJ80cN+iS85XCXUQyV1oDH7nxwHUN73nhPsi++Gt//ipvbGoEoEgt86zR\nSH8RGZqeN00Nwp72LgBuu+jYg04pIAOncBeRodkb7i/eMajZKIsKwpxz1Hj+dMGU/neWjCncRWRo\nCmKAwZ4N0DzwRzk0dXRrFMwwULiLyNB95j5vOcCumR3NnWxt6mRMkcI92xTuIjJ0e7tmBnhR9c30\nKJlZ1aXZrijvKdxFZOj2TlbWuh262j58k9NBNHV449tPPbxquCrLWxoKKSJDV5SeqOvXV3rLUAH8\n2TMw/qhDfltjeqRMubplsk7hLiJDVzEFPnUntO2A1h3w4g9h17p+w725oxuzvm9ekqHRn6iIZMdx\n6SdyNm7ywr2jsd9vaUyPlAlpFsesU5+7iGTX3v73DC6uNnV0U1GsLpnhoJa7iGRXYak3/8yqJd4F\n1t4mzodjLgKgsb1b/e3DROEuItllBtMXweZXYceqA7clOiFWQd3Exazb0Ur9nnYmVhT5U2fAKdxF\nJPsuX9L3+qU3w6v3cuW9y3m/oQ2ABdP0SLzhoHAXkZETiUKikx3xOJ+cN5GrFs1g9viy/r9PBkwX\nVEVkxCyra4VUgvZ4nFnVpRw7uUIP4BgmCncRGTFPvd8CQJRuKkt0IXU4KdxFZEQkkinieIEepYsp\nY4t9rijY1OcuIiNiW3MncQoB+N2XT2TitJp+vkOGQi13ERl2XYkUp976FHHntdwnluqO1OGmlruI\nDLtdbd4skVNqxkIj8PQtUNxjCOSM0+CI8/wpLqAU7iIy7Ha1erM/nnTSKbBsAqxdun9jVzusf1rh\nnmUZhbuZnQvcDoSBu51zt/TaXg78HJia/szvOed+muVaRSRH7Wz1Wu6xCXPhL1cfuHHJDfDe4z5U\nFWz99rmbWRi4A1gMzAU+Z2Zze+12HfCuc24ecDrwfTMrzHKtIpKjdrd5LfdxpdEPb4yVD/gJTtK/\nTFruC4F1zrn1AGZ2P3AB8G6PfRxQZmYGlAK7gUSWaxWRHLW3W2ZcaR9tvlg5JDqgZRuEe20Phfc/\nwk8GJJNwnwRs6vG+Hjip1z4/BJYAW4Ay4LPOuVRWKhSRnLarNc7KLU0UhI2yaB+RU5x+xN735/T9\nARfdA0dfOHwFBlS2LqieA7wBfAyYBTxhZs8555p77mRm1wDXAEydOjVLhxaR0ewr97/OC+t2MX1c\nMd4v970cfSHgINl94PpkNzx+MzRuHJE6gyaTcN8MTOnxfnJ6XU9fBG5xzjlgnZltAI4AXum5k3Pu\nLuAugAULFrjBFi0iuWPT7g4W1VZx64XH9r1DbAwsuPLD65MJL9x7h75kJJObmJYDtWY2I32R9GK8\nLpieNgJnApjZeGAOsD6bhYpI7nHO0dASZ874soHP2x4KA6ZwH6R+W+7OuYSZXQ8sxRsKeY9zbqWZ\nXZvefifwbeBeM3sbMOAbzrmdw1i3iOSAtq4kHd1Jqsv6GCXTHzMIF0CyK/uF5YGM+tydc48Bj/Va\nd2eP11uAj2e3NBHJdTtbvPHtgwp38EbPpDTwbjA0t4yIDJuG1iGGeyiilvsgKdxFZNg0pFvuVX3d\nvJSJcKH63AdJ4S4iw6ZhyN0yBQr3QVK4i8iwaWiJEw4ZlcWDnI0kXAAphftgaFZIERkWNz34Fg+s\n2ERVaZRwaJDzt4c0WmawFO4iMiyWvb+L2poy/uLs2YP/EPW5D5q6ZUQk65xzbGvu5LQ51Zx79GGD\n/6BwROE+SAp3Ecm6po5uuhIpagZ7IXWvkPrcB0vhLiJZ1djexb3LPgBg/JjY0D6soMh7UpMMmMJd\nRLLq9j+s5QdPrgXgsPIhhnusHOLN/e8nH6JwF5Gs2pEe2w4wbWzx0D4sVqGnNA2Swl1Esub9hlZ+\n99ZWjjisjOU3n0XNULtliiqgeTM4zRA+UAp3EcmaZeu8yWAvPH7y4O9K7al4nLf8338c+mflGYW7\niGRNc6c3g+Olp0zLzgeecIW33K3HQwyUwl1EsqalM0FB2IhGshQtxWNh0gLobMzO5+URhbuIZE1L\nZzdlsYK+n5U6WLFyXVQdBE0/ICJZ09KZoCyW5ViJlUP9Cnjkeu99JAqnfQNKa7J7nIBRuItI1jS0\nxBlXMsgZIA9m5umw6WVY9wdwSWjdDhOPh/mXZPc4AaNwF5GsqW9s5/ipldn90BMu974AOvbArdPV\nB58B9bmLyJAlU476Pe1sbexkcmXR8B0oWg6Y+uAzoHAXkSG7+7n1nHrrUyRSjkkVQ7wr9VBCIYiN\ngabN0Lpj+I4TAAp3ERmyl9bv2vd62rhhDHeAkmp44+fwvVpY+8TwHiuHKdxFZMhiBWEA/u1z8zl5\n5rjhPdiFP4GP/5P3uql+eI+VwxTuIjJkG3e3c8acav543sTBP1IvUxOPg+M+771OxA+9bx5TuIvI\nkKRSjo272pk61BkgByKSnpAs0Tlyx8wxCncRGZLLf/oKLfEE87M9BPJQIulJyRTuB6VwF5EhWbW1\nhXDIhvas1IEKhb1H8CncD0rhLiKD5pyjqaOLqxfN3HdRdcREYupzPwSFu4gMinOObc2ddCcdlcUF\nI19AQUwt90PQ9AMiMig/fuZ9vvs/awAYV5qFB3MMVCQG3Qr3g1G4i8igvLethXElhXzt47NZPJL9\n7XtFopDoGPnj5oiMumXM7FwzW2Nm68zspoPsc7qZvWFmK83smeyWKSKjze72biaPLeaSk6ZREvWh\nnRgdA53NI3/cHNHvT8TMwsAdwNlAPbDczJY4597tsU8F8CPgXOfcRjPTRMsiAbenrYuq0ixP7zsQ\nRRWaQOwQMmm5LwTWOefWO+e6gPuBC3rt83ngN865jQDOOc3oIxJw25o7qfKjr32vWLmm/j2ETMJ9\nErCpx/v69LqeZgOVZva0mb1qZpdlq0ARGV1WfLCbG+9/nYaWOIfXlPpXSKzCmx3y8b+DVMq/Okap\nbA2FjAAnAOcD5wDfNLPZvXcys2vMbIWZrWhoaMjSoUVkJP3ylY387q2t1NaU8tHZ1f4VMusMb/rf\nZf8GTRv9q2OUyuQqyGZgSo/3k9PreqoHdjnn2oA2M3sWmAe813Mn59xdwF0ACxYscIMtWkT88cS7\n23lu7U6OmjiGR64/1d9i5l4AFoZfXaK+9z5k0nJfDtSa2QwzKwQuBpb02ucR4FQzi5hZMXASsCq7\npYqIn3a3dXH1fSu856T62dfeU6zcW3ao7723flvuzrmEmV0PLAXCwD3OuZVmdm16+53OuVVm9j/A\nW0AKuNs5985wFi4iI2v11v3DDof1UXoDUVThLdVy/5CMBqc65x4DHuu17s5e728DbsteaSIymtTv\n8W4Y+uHn53PmEeN9riZtb8tdo2Y+RHeoikhGGlq9SbrOOnL8yE8SdjAxtdwPRhOHiUhGfvnyRkoK\nw6Mn2AGiZWAh9bn3QeEuIv3q7E6ypamDaeNK/C7lQGbpm5nUcu9N4S4i/VqzrQXn4IaPHe53KR8W\nK4fl/6GA70XhLiL9uuOpdQAcPanc50r6UDPXW256xd86RhmFu4gcUmd3ksff3U5lccHoGQLZ01nf\n8pZquR9A4S4ih7R05TYArjvjcMzM52r6oOGQfVK4i8gh7R3ffslJ03yu5CD2hntT71lR8pvCXUQO\nqr0rwW1L11AWjVBUOIqGQPZUEPOWr93nbx2jjMJdRA7q9Y1eV8exU0bhhdSeJi+EkO7J7EnhLiIH\n9d72FgD++dPH+FxJP6YshLgeudeTwl1E+vS/q7fzrUe9p2mOHxPzuZp+xCqgux0SXX5XMmoo3EWk\nT0+t9h6o85PLF4yuKQf6snd2yAcug1f+w99aRgmFu4j0afkHu1lUW8WZR46SGSAPZerJcNixsHEZ\nPP8Dv6sZFRTuInKA1niC/3zxA9Zsb+HE6WP9Liczhx0D1z4Hx+mpTHsp3EXkAP/18ka++chKAE7z\n8xmpgxGrgK4WSCb8rsR3GjskIgd4YtV2ZlSV8OgNp1IazbGI2He3ahOUjPO3Fp+p5S4i+7y5qZFX\nNuzmpBljcy/YAYrT3Ujv/NrfOkYBhbuI7POHVdsBuPa0WT5XMkhHfMJb6uEdCncR2e+ZtTuZP7WC\n6VWj7KEcmSoshsJSXVRF4S4iaXvaunirvjH3LqL2FqvQDJEo3EUk7bl1O3EuB0fI9BYrhzfv97sK\n3yncRQSAZ9Y0UFFcwLGTK/wuZWgKi8Eloavd70p8pXAXEZxzPLu2gUW11YRDo/CBHAMx73PeMs/7\n3RXuIsLqbS00tMRzv0sG9s8zk+fhnoMDWUUk21Zu8abLPX5qjnfJwP4bmTa/Csk4WBhqjoTQKJ/8\nLMsU7iLC+w2tFISNqWOL/S5l6EoP85aP/Pn+ded9DxZe7U89PlG4iwiv1e1h2rgSIuEA9NSOPwou\n/+3+h3c8cDk01ftbkw8U7iJ5rm5XGy9v2M05R+XA1L6ZMIMZi/a/L8rPce8B+G9aRIbit29tBeDL\npx/ucyXDJFaRlxdXFe4ieeydzU3ctnQNJ0yr5LgpAbiY2pdYOax8CJzzu5IRpXAXyWPf+f0qAC47\nZZrPlQyj2Bhv2b7b3zpGWEbhbmbnmtkaM1tnZjcdYr8TzSxhZhdlr0QRybZ3Njex8J+e5IV1u/jK\nmbVccNwkv0saPsd8xlvmWb97v+FuZmHgDmAxMBf4nJnNPch+twKPZ7tIEcmu/3yxjtZ4gqsXzQh2\nqx163NSkcO9tIbDOObfeOdcF3A9c0Md+NwAPAjuyWJ+IZJFzjjueWsevVmzivGMmcPP5c6kqjfpd\n1vDq+XSmPJJJuE8CNvV4X59et4+ZTQI+Dfw4e6WJSLZtbuzgtqVrALj05IC32PeKpVvuefYAj2xd\nUP0B8A3nXOpQO5nZNWa2wsxWNDQ0ZOnQIpKp7z/+HgAPX/cR5gV1dExvedpyz+Qmps3AlB7vJ6fX\n9bQAuN/MAKqA88ws4Zx7uOdOzrm7gLsAFixYkF/jkkR81hZP8NDrmxlXUsi8yeV+lzNy8nQisUzC\nfTlQa2Yz8EL9YuDzPXdwzs3Y+9rM7gV+2zvYRcQ/r2/cw6d/tAyAG8+qJd0Qyw8FxRCK6IJqb865\nBHA9sBRYBTzgnFtpZtea2bXDXaCIDN0rG7wx3n99zhwuO2W6v8WMNDOv3/3tX8N/fR721Pld0YjI\naG4Z59xjwGO91t15kH2vGHpZIpJN721vZWxJIdedEdApBvoz/wvw/h9gze9gzmKovNTvioad7lAV\nCbhkyvH4ym0sqq3yuxT/nP0tuCLdPs2T7hmFu0jA/fSFDbTEE5w+JwBPWRqKwlKwUN5cWFW4iwRY\nVyLFd36/GoDFR0/wuRqfhUIQLYNNL/tdyYhQuIsEVGs8wZX3LieZcnzp1BnECvLrMXN9isRgw7Ow\nfaXflQw7hbtIQP38pTqeX7eTGVUlfOXMWr/LGR3O+Wdv2bzV3zpGgJ7EJBJAy97fyS2/X83cCWN4\n7MZF/X9DvjjsWG+ZBxdV1XIXCZhEMsXf/OZtAL5+7hyfqxll9k1FoHAXkRzzjQffpm5XO9d8dCan\nz6nxu5zRZW+4v/gjWBnsm+gV7iIBsqs1zoOv1XP6nGq+dvZsv8sZfQpicPxl0LIN3vqV39UMK/W5\niwSEc44v3rscgBvPrNXomIP55L/B7g2BnwJYLXeRgHh3azNv1Tfx1bNqmT+10u9yRrdYeeBvZlK4\niwSAc45L7n6Z4sJw/k0MNhixisBfVFW4iwTAG5saaWzv5sTpYxlbUuh3OaNfUYVa7iIy+j36pndT\nzm0XHetzJTkiVg5drZBM+F3JsFG4i+S4ul1tPLBiE5+cN5GaMTG/y8kNefDoPYW7SA5rau/mU3e8\nQFcylb9ztQ/G3odmB7jfXeEuksNuuP919rR3c+cXjmfOYWV+l5M79rbcH/wS/OyPYf3TvpYzHBTu\nIjmqoSXOqx/s5uhJY/jYEeP9Lie3TDoBaj/uPV+17kVY9ajfFWWdbmISyUHPr93JV3/1BknnuO2i\neX6Xk3tKq+GS//Ze335cIPveFe4iOebVuj1cee9yQiG48wsncOSEMX6XlNsCekOTwl0kRzjneGDF\nJm5/ci3VZVF+e8OpVGpM+9DFyqGpHupfhUnHg5nfFWWF+txFcsRTa3bwjQffZktTJ9/8xJEK9mwZ\nMwl2vAt3fwzqlvldTdao5S6SI+57sY7xY6I8/VdnUFSoScGyZvEtMPM0eOjPoCU4T2hSy10kBzjn\neGdzE6ceXq1gz7ZYOcw83XsdoL53hbtIDqjf08HO1i6OnqSLp8MigDc1KdxFRrmdrXGuvHc50UiI\nRbXVfpcTTAUxCEdh+T2w5AZwzu+KhkzhLjKKfbCzjat+toK6Xe3c+8WFHF5T6ndJwXXytRAphNfu\ng+52v6sZMl1QFRmFnHPc8dQ6vvf4ewD89TlzOGXWOJ+rCriz/y9UzoDfftXrey8s8buiIVG4i4wy\n/71iE798ZSOvb2zkhGmV/N35R+rJSiNl75wzHY0wZqK/tQyRwl1kFPmLX73BQ69vJhwybj7vSK5a\nNAMLyE01OaEofWF1w7Mwfq6/tQyRwl1klHjmvQYeen0zn5w3ke9/Zh4FYV0SG3FjJnnL//kGzDkX\nKqf7Ws5QZPS3x8zONbM1ZrbOzG7qY/slZvaWmb1tZsvMTDMZiWSoblcb19y3gsvveYWyaISbFh+h\nYPdL9Rw4+9ve65bt/tYyRP223M0sDNwBnA3UA8vNbIlz7t0eu20ATnPO7TGzxcBdwEnDUbBIkPzn\nS3X8nyUrCZtx5hE13Pan8/QMVL9N+4i3zPEx75l0yywE1jnn1gOY2f3ABcC+cHfO9ZyQ4SVgcjaL\nFAmapvZuvvbAG/xh9Q7mT63gXy+ez5SxxX6XJRCYR/Bl8rvfJGBTj/f16XUH8yXg90MpSiTI3tzU\nyHn/+hx/WL2DE6ZV8ourTlKwjyZ7L6r+5mrYU+dvLUOQ1QuqZnYGXrifepDt1wDXAEydOjWbhxbJ\nCT9/qY5v//ZdyosK+PdLT+Ccow7zuyTprXic1zVT9wJseQ0qp/ld0aBk0nLfDEzp8X5yet0BzOxY\n4G7gAufcrr4+yDl3l3NugXNuQXW1bqOW/HL7k2v5u4ff4eSZ4/jdVxYp2EcrM7jgh97rRNzfWoYg\nk5b7cqDWzGbghfrFwOd77mBmU4HfAJc6597LepUiOcY5x+62Lna2drF2Rwv3LavjlQ9284ljJ/CD\nzx5HRKNhRrdIzFt2d/hbxxD0G+7OuYSZXQ8sBcLAPc65lWZ2bXr7ncDfA+OAH6VvuEg45xYMX9ki\no9Pmxg5++XIdT7y7nfe2t+5bP7myiCv+aDo3LT5CwZ4L9oZ7wFvuOOceAx7rte7OHq+vAq7Kbmki\no18imeLfn13PC+t2squ1izXbWwCorSnlrz4+m2njSvbN5qh52HNIJOotE53+1jEEukNVZBDauxL8\n9IUPuPOZ92npTHDUxDGMLSnka2fP5iOHV3HCNM0Fk9PypeUuItAWT/B+QysrtzTzz4+toqUzwcIZ\nY7n8lOmcf+wEv8uTbAqFIVQAiQD3uYvkq1TKsaJuD4++uYXn1+1kw862fdtmjy/lrkuP1jS8QRaJ\nqeUuEhTOOV7ZsJufPL+BF9/fRUs8QawgxB/NquJP5k9iWlUJM6tKmD2+jMKILowGWrgA3rwfzv2O\n35UMisJdJO3h1zfzD0tW0tTRTXVZlE/Mm8gps8Zx5hE1lET1TyXvFJbm9Pwy+hsreaupvZvV25p5\ne3MTD72+mZVbmjlhWiUXHj+ZT82fSHGh/nnktXmfhWe/B6kUhHLvtzT97ZW8kEim2NES59E3t7B0\n5Tb2tHfzwa62fc9Bnjelgr//xFy+cPI0dbeIJ1YOOOhq2T+ZWA5RuEtgbNjZxtbGDt7Z0sT7O9pY\nvb2F3W1x2uJJdrd17dvv2MnlzJ0whk8dN4n5UyuYXFnEzGo9eFp6iaUnEFv1KBRX7V8fLoAZH/WW\no5jCXUa1ZMrR3NHNpj3txBMp9rR18c6WZup2tdGdTNGVcOxqi/PethbaupL7vq+6LMrEiiIWTBtL\nUWGYmrIoFUUFnFpbzeE1CnLJQHl65vJHrvvwtgt/AsdcNLL1DJDCXUZcWzzBpj3trNnWwpbGTlo6\nu9nV2kVDa5zG9i66kil2tXaxvbmTlOv7MyaWxyiJRigIhxhTFOFPF0xhRlUJtTWl1I4vo7osOrIn\nJcEz83T485cOnF+mux3uPR9atvlVVcYU7pI18USS5o4Ee9q7aI0nqNvVxramOPV72vlgVxt1u9pp\niyfY0959wPeFQ8bYkkKqS6NUlhRQHi5g2tgSpo0rJhoJEy0IMX1cCaXRCKWxCLU1pRq9IsPPDGqO\nPHBdKgVYTjzIQ/9C8lwimWJnaxfbmjtJJFMkUo54IkVzRzctnQkaWuIkU976ZMrREk/Q2Z0kmXIk\nUo7WzgRt8QTbWzrZtLvvu/kqiguYNraYBdMqKY1FGFcSZVZNKdPHFTOrupTiwjDpCedERrdQCGJj\ncmKIpMJ9lEmmHB3dSVo7E7TGu2mNe6/buxLe+niCjq4k8USKeCJFZ3eStniC7mSKRNLRlV7GE0na\nupJ0JVLeV9Jb7ts3Hc5diVS/NYUMIqEQ4ZBRXBgmVhAmEjYiIaO4MMKYogjzJldw4fGTGVdSSEVx\nIaXRCJMri5hYUaRWtgRLrFwt96BJpluqSedIOUcq5Ug5SDmvVesc+7Z1J1M0tMRpiydojSfp6Ers\nC+SWzgRbmzrY095NZ1eSti6vhdzY0Z1R2PYUKwhRmu57joSNgnCIglCIwkiIkmiYMUUFFIaNwkiI\ngnCIkmiEwnCISMgIh42igjBjYgVMryomEvLWF0ZClBcVUBKNUFMW1RS1Ij3FKqBDLfese2dzE79+\ntX5f10AyHbJJ580FsnddIpmiszvVY3s6iNP7pPaFsCPenSSVDmaX3m/fPj0CvCuZ2jcueigKwsZh\n5THGlkQpLggzfkyMoyaOobKkkOKCCEWFXgiX9vgqiUYoKgxTUhihOBomGglRGA6pO0NkpKnlPjw2\nN3bw4Kv1FBWGiYSMUMgImREOGSEjvfRasLECryshEgntW99zn5AZBZEQsfR2S2/f+3nW63U0EmZM\nLOK1evftb4RDYGaEzQiF9n9PdWmUslgBJdEwxYURYgUhopEwBWFTKIvkqqIK2LnW7yr6lXPhfs5R\nh3HOt/TsSRHxSawcGlZDZ7N3cXWUUmeqiMhAlE/xli/+0N86+qFwFxEZiEV/5S1bt/tbRz8U7iIi\nAxGOwLjaUX9RVeEuIjJQRaN/OKTCXURkoGLlsP4p2Pqm35UclMJdRGSgjvoTb1m3zN86DkHhLiIy\nUMd+1ltRC/52AAAFpklEQVSO4n53hbuIyECFI1BYNqr73XPuJiYRkVGhqAK2vwMrH878e0rHw7RT\nhq+mHhTuIiKDUT4FPnjO+8qYwdfXQ/HYYStrL4W7iMhgXPIANG7KfP91T8IT34S2nQp3EZFRK1oG\n4+dmvn/zFm85Qg/60AVVEZGRECv3liN0EVbhLiIyEooqvOWjN8Ky4Z90TN0yIiIjoXIGnHgVtDVA\nac2wHy6jcDezc4HbgTBwt3Pull7bLb39PKAduMI591qWaxURyV3hCJz//RE7XL/dMmYWBu4AFgNz\ngc+ZWe+rCIuB2vTXNcCPs1yniIgMQCZ97guBdc659c65LuB+4IJe+1wA3Oc8LwEVZjYhy7WKiEiG\nMgn3SUDPwZz16XUD3Qczu8bMVpjZioaGhoHWKiIiGRrR0TLOubuccwuccwuqq6tH8tAiInklk3Df\nDEzp8X5yet1A9xERkRGSSbgvB2rNbIaZFQIXA0t67bMEuMw8JwNNzrmtWa5VREQy1O9QSOdcwsyu\nB5biDYW8xzm30syuTW+/E3gMbxjkOryhkF8cvpJFRKQ/GY1zd849hhfgPdfd2eO1A67LbmkiIjJY\n5uWyDwc2awDqfDm4pwrY6ePxR0o+nGc+nCPoPINkKOc4zTnX74gU38Ldb2a2wjm3wO86hls+nGc+\nnCPoPINkJM5RE4eJiASQwl1EJIDyOdzv8ruAEZIP55kP5wg6zyAZ9nPM2z53EZEgy+eWu4hIYOVV\nuJvZcWb2kpm9kZ7AbGGPbX9jZuvMbI2ZneNnndlgZjeY2WozW2lm3+2xPlDnCWBmf2lmzsyqeqwL\nxHma2W3pn+NbZvaQmVX02BaIc9zLzM5Nn8s6M7vJ73qyxcymmNlTZvZu+t/jjen1Y83sCTNbm15W\nZvXAzrm8+QIeBxanX58HPJ1+PRd4E4gCM4D3gbDf9Q7hPM8AngSi6fc1QTzP9DlNwbt7ug6oCtp5\nAh8HIunXtwK3Bu0c0+cTTp/DTKAwfW5z/a4rS+c2ATg+/boMeC/98/sucFN6/U17f7bZ+sqrljvg\ngDHp1+VA+nHkXADc75yLO+c24E2jsLCP788VXwZucc7FAZxzO9Lrg3aeAP8P+Drez3avwJync+5x\n51wi/fYlvEn5IEDnmJbJcyNyknNuq0s/mc451wKswpsS/QLgZ+ndfgZ8KpvHzbdw/ypwm5ltAr4H\n/E16fUbz0eeQ2cAiM3vZzJ4xsxPT6wN1nmZ2AbDZOfdmr02BOs8ergR+n34dtHMM2vn0ycymA/OB\nl4Hxbv8Ei9uA8dk8VuAekG1mTwKH9bHpZuBM4C+ccw+a2WeAnwBnjWR92dLPeUaAscDJwInAA2Y2\ncwTLy5p+zvNv8botctqhztE590h6n5uBBPCLkaxNssfMSoEHga8655q9R097nHPOzLI6dDFw4e6c\nO2hYm9l9wI3pt/8N3J1+nXPz0fdznl8GfuO8zrxXzCyFN5dFYM7TzI7B62t+M/2PZDLwWvoieU6d\n56F+lgBmdgXwCeDM9M8UcuwcMxC08zmAmRXgBfsvnHO/Sa/ebmYTnHNb048l3XHwTxi4fOuW2QKc\nln79MWBt+vUS4GIzi5rZDLwHfb/iQ33Z8jDeRVXMbDbeBaqdBOg8nXNvO+dqnHPTnXPT8X6NP945\nt40AnaeZnYt3TeGTzrn2HpsCc45pmTw3IieZ1/r4CbDKOfcvPTYtAS5Pv74ceCSbxw1cy70fVwO3\nm1kE6ASuAXDe/PQPAO/i/ep7nXMu6V+ZQ3YPcI+ZvQN0AZenW3xBO88+Bezn+UO8ETFPpH9Deck5\nd23AzhF3kOdG+FxWtnwEuBR428zeSK/7W+AWvC7TL+GN9vpMNg+qO1RFRAIo37plRETygsJdRCSA\nFO4iIgGkcBcRCSCFu4hIACncRUQCSOEuIhJACncRkQD6/+BwknC87QilAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(thresholds, precisions)\n", + "plt.plot(thresholds, recalls)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Precision-Recall 曲线" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE9tJREFUeJzt3X+MXeV95/H31zO2sY3BP/llezIumFJXwQEmJAtsliRK\ni8m2blZJC6kShaa12IbsaldaBaWrVqtUaqpspG5aEtdCNGq7ClUSkjjFhERKCmmIW8wGMA4/OnEA\n2wP4N7bHYw8z890/5hLGg8f3zsyde+88835JI80555lzvjwaPn7muec8JzITSVJZZjW7AElS/Rnu\nklQgw12SCmS4S1KBDHdJKpDhLkkFMtwlqUCGuyQVyHCXpAK1N+vCy5Yty87OzmZdXpKmpccee+xA\nZi6v1q5p4d7Z2cn27dubdXlJmpYi4oVa2jktI0kFMtwlqUCGuyQVyHCXpAIZ7pJUoKrhHhH3RMS+\niHhqjOMREV+IiO6IeDIirq5/mZKk8ahl5P5l4KazHF8PrKl8bQS+NPmyJEmTUfU+98x8OCI6z9Jk\nA/C3Ofy+vm0RsSgiLs7Ml+pU42meffkY9z/ZMxWnLkbH0gV88JqVzS5DUhPV4yGmFcDuEdt7Kvve\nFO4RsZHh0T0dHR0Tulj3vuP85Q+6J/SzM8Hrr8T9rbddQnubH6lIM1VDn1DNzM3AZoCurq4JvZn7\n/VdezPuvfH9d6yrJ57/7LHf9oJu2WdHsUiQ1UT2GdnuBVSO2V1b2qQlOvjbI3PY2Igx3aSarR7hv\nAT5auWvmncCrUzXfrupODQwxd/YsBgaHxvzKnNAfTZKmkarTMhHxFeBGYFlE7AH+BJgNkJmbgK3A\nzUA3cAK4baqKVXUDQ8mRE69x2R89MGabdasW8a1PXN/AqiQ1Wi13y9xa5XgCn6hbRZqUj9+wmovP\nO2fM4z94dh//9srxBlYkqRmatuSvpsaly8/lk+9dM+bxwyde4znDXSqe98rNMBE45y7NAI7cZ5gA\n+geH+NyDzzS7lKLMiuBD16yiY+n8ZpciAYb7jHP5RQsJgr9+aFezSynGUCZDCYvnz+H3bljd7HIk\nwHCfcX67axW/3bWqekPV7H99eyd/v+0F/uOVFze7FOkXnHOXJuH4qQG+un0P73/rxVxwlruUpEZz\n5C5Nwtcf28PxUwPcem0HpwYG63be2bNmMcslJDQJhrs0Cd9+YniF0t/ZvK2u572qYxH3/efrXEZC\nE2a4S5Pw33/tcn7y4pG6nvO5V47xrcd7ePqlY6y95Ly6nlszh+EuTcJ1ly7jukuX1fWch3r7uf/J\nl9jyRI/hrgnzA1WpxSxZMId/v2YZ336ih6EhHzjTxDhyl1rQb77tEv7bPzzBH31zB+fNm33GNhvW\nrXBkrzEZ7lILet/ai3jL0n/jvv/35lcjDGXy2uDwiN5w11gMd6kFnTu3nYf+x7vPeOzuH+7iT+9/\nmt9cd0mDq9J04py7NI0MDSX/919e5OqORfzqJec3uxy1MEfu0jTyyM8O8vMDvfzBB97Kif6Bupyz\nbVYwt72tLudS6zDcpWnkgaeG32D56W/s4NPf2FGXc85tn8X9/+UGLrtgYV3Op9ZguEvTyG3Xd9Kx\npH7LCm95oofnXjnGkgVz63ZOtQbDXZpGLrtgYd1G2P0DQ2x+eBfvW3shSxbMqcs51Tr8QFWaob7/\nzD4O9vbzoWtcArpEjtylGeprj+0G4KHn9vOj7gNNrubsVi6ex8eu90Uo42G4SzPU3NltLJjTxle3\n7252KWd1cmCItgg+8u86aXMZ5JoZ7tIMddeHr252CTX5jb/8Z+bNbjPYx8k5d0kt68iJfp7qeZXr\nLlva7FKmHcNdUsvatusgmXD9ZfVdVnkmMNwltaxHfnaQ+XPaWLdyUbNLmXYMd0kt60fdB3h75xLm\ntBtV42WPSWpJL796kp/t7+V659snxHCX1JJ+vGv43vt6v8ZwpvBWSEkt6UfdBwH46vbdfO2xPQ25\n5uL5c7jjPZcVcdul4S6pJS07dy7nndPON37y5rdRTYVTA0OcGhjiQ10ruWTRvIZccyrVFO4RcRPw\nf4A24O7M/Oyo4+cDfw90VM75vzPzb+pcq6QZ5M71V3Dn+isadr0/e+Bp/uafn+fC885p2DWnUtU5\n94hoA+4C1gNrgVsjYu2oZp8AfpqZ64Abgc9HhMvMSZo2nj/Qy6ol84qYkoHaRu7XAt2ZuQsgIu4F\nNgA/HdEmgYUREcC5wCGgPq+JkaQGeOHgCS5ZNI/Dvf1jtjl/3mxmTZPwryXcVwAjVxbaA7xjVJu/\nArYAPcBC4Hcyc6guFUpSA7x89CTPvHyMqz7zvTHb3HrtKv7sP13ZwKomrl4fqP468DjwHuBS4HsR\n8cPMPDqyUURsBDYCdHR01OnSkjR5X7jlKnbtPz7m8c9/9zn6+gcbWNHk1BLue4GRq/mvrOwb6Tbg\ns5mZQHdE/By4AvjXkY0yczOwGaCrqysnWrQk1du7Ll/Ouy5ffsZjg0PJn97/9LS6i6aWh5geBdZE\nxOrKh6S3MDwFM9KLwHsBIuJC4JeBXfUsVJKa5cDxUwwM5bQK96oj98wciIg7gAcZvhXynszcGRG3\nV45vAj4DfDkidgABfCozW/vVLpJUo71H+gBYUVK4A2TmVmDrqH2bRnzfA/xafUuTpNbQUwn36TRy\nd20ZSarijXCfPg84Ge6SVEXPkZMsnNvOwnNmN7uUmhnuklTF3iN902pKBgx3SarqpVf7ptWUDBju\nklRVz5GTjtwlqSR9/YMc6u033CWpJD2vTr87ZcBwl6Sz+sVtkOc7cpekYkzHB5jAcJeks+o5cpII\nuOh8p2UkqRg9R/q4cOE5zG6bXnE5vaqVpAbrebWPi6fZh6lguEvSWe09PP2eTgXDXZLGNDiU7D3S\nx6rF85tdyrgZ7pI0hpePnuS1weQtSw13SSrGCwd7AehYMv3CvV4vyJak4uw+dAKAv/vxC2x5vOes\nbX/l4oV87PrVjSirJoa7JI1h1ZL5rFg0j8d3HzlruyN9/XxnZ5vhLknTwXWXLuNHd76nartPfuUn\n7Nhz9n8AGs05d0mapEO9p1iyYE6zyziN4S5Jk3TweD9LFsxtdhmnMdwlaZIO9fazZEFrvV/VcJek\nSchMDp9w5C5JRTl2aoDXBpOlzrlLUjkOHe8H8ANVSSrJoROGuyQVx5G7JBXoUK/hLknFeX1aZum5\nhrskFeNQbz/nzJ7F/DmttZqL4S5Jk3DweD9L5rfWqB1qDPeIuCkino2I7oi4c4w2N0bE4xGxMyIe\nqm+ZktSa9h07yfLzWu8dq1X/joiINuAu4H3AHuDRiNiSmT8d0WYR8EXgpsx8MSIumKqCJamV7Dt6\nqiXf1FTLyP1aoDszd2VmP3AvsGFUmw8D92XmiwCZua++ZUpSa9p37CQXnNdaSw9AbeG+Atg9YntP\nZd9IlwOLI+KfIuKxiPhovQqUpFZ1amCQwyde48KF03BaZhznuQZ4LzAP+HFEbMvM50Y2ioiNwEaA\njo6OOl1akppj/7FTANN25L4XWDVie2Vl30h7gAczszczDwAPA+tGnygzN2dmV2Z2LV++fKI1S1JL\neOXo6+HeeiP3WsL9UWBNRKyOiDnALcCWUW2+BdwQEe0RMR94B/B0fUuVpNay/9hJgOk5LZOZAxFx\nB/Ag0Abck5k7I+L2yvFNmfl0RHwHeBIYAu7OzKemsnBJarY3Ru6tNy1T05x7Zm4Fto7at2nU9ueA\nz9WvNElqbfuOnaR9Vkzfh5gkSW/2ytFTLF84l1mzotmlvInhLkkT9MrRky35YSoY7pI0YfuPneKC\nha033w6GuyRN2CtHT3JhC36YCoa7JE3IydeGn069yGkZSSrH3iN9AKxYPK/JlZyZ4S5JE7D3cCXc\nF7XeipBguEvShDhyl6QC7T3cR9us4ELvlpGkcvQc6eOi886hva01Y7Q1q5KkFrfnSB8rFrXmlAwY\n7pI0IXsP97XsfDsY7pI0bgODQ7x89KQjd0kqyUuvnmRwKFnpyF2SyvH8wV4AOpctaHIlYzPcJWmc\nnj8wHO6rDXdJKsfzB08wb3Zby64ICYa7JI3b8wd6ecvS+US03ks6Xme4S9I4PX+wl86lrTslA4a7\nJI3L4FCy+1BfS3+YCoa7JI1Lz5E++geH6FzamqtBvq692QVI0nTy+m2Q/7B9N99/Zt+4f35WBLff\neClvW7Wo3qWdxnCXpHFYtXg+V3Usoq9/kBcPnRj3zz/z8jE6ly0w3CWplXQuW8A3/vD6Cf/85f/z\ngTpWMzbn3CWpQIa7JBXIcJekAhnuklQgw12SCmS4S1KBDHdJKpDhLkkFqincI+KmiHg2Iroj4s6z\ntHt7RAxExAfrV6IkabyqhntEtAF3AeuBtcCtEbF2jHZ/Dny33kVKksanlpH7tUB3Zu7KzH7gXmDD\nGdp9Evg6MP6VdCRJdVVLuK8Ado/Y3lPZ9wsRsQL4APCl+pUmSZqoen2g+hfApzJz6GyNImJjRGyP\niO379++v06UlSaPVsirkXmDViO2VlX0jdQH3Vt4nuAy4OSIGMvObIxtl5mZgM0BXV1dOtGhJ0tnV\nEu6PAmsiYjXDoX4L8OGRDTJz9evfR8SXgX8cHeySpMapGu6ZORARdwAPAm3APZm5MyJurxzfNMU1\nSpLGqaaXdWTmVmDrqH1nDPXM/Njky5IkTYZPqEpSgQx3SSqQ4S5JBTLcJalAhrskFchwl6QCGe6S\nVCDDXZIKZLhLUoEMd0kqkOEuSQUy3CWpQIa7JBXIcJekAhnuklQgw12SCmS4S1KBDHdJKpDhLkkF\nMtwlqUCGuyQVyHCXpAIZ7pJUIMNdkgpkuEtSgQx3SSqQ4S5JBTLcJalAhrskFchwl6QCGe6SVKCa\nwj0iboqIZyOiOyLuPMPx342IJyNiR0Q8EhHr6l+qJKlWVcM9ItqAu4D1wFrg1ohYO6rZz4H/kJlv\nBT4DbK53oZKk2tUycr8W6M7MXZnZD9wLbBjZIDMfyczDlc1twMr6lilJGo9awn0FsHvE9p7KvrF8\nHHhgMkVJkianvZ4ni4h3MxzuN4xxfCOwEaCjo6Oel5YkjVDLyH0vsGrE9srKvtNExJXA3cCGzDx4\nphNl5ubM7MrMruXLl0+kXklSDWoJ90eBNRGxOiLmALcAW0Y2iIgO4D7gI5n5XP3LlCSNR9Vpmcwc\niIg7gAeBNuCezNwZEbdXjm8C/hhYCnwxIgAGMrNr6sqWJJ1NTXPumbkV2Dpq36YR3/8+8Pv1LU2S\nNFE+oSpJBTLcJalAhrskFchwl6QCGe6SVCDDXZIKZLhLUoEMd0kqkOEuSQUy3CWpQIa7JBXIcJek\nAhnuklQgw12SCmS4S1KBDHdJKpDhLkkFMtwlqUCGuyQVyHCXpAIZ7pJUIMNdkgpkuEtSgQx3SSqQ\n4S5JBTLcJalAhrskFchwl6QCGe6SVCDDXZIKZLhLUoEMd0lqoJt+9SKuuGjhlF+npnCPiJsi4tmI\n6I6IO89wPCLiC5XjT0bE1fUvVZKmvy/cehW/ddWKKb9O1XCPiDbgLmA9sBa4NSLWjmq2HlhT+doI\nfKnOdUqSxqGWkfu1QHdm7srMfuBeYMOoNhuAv81h24BFEXFxnWuVJNWolnBfAewesb2nsm+8bYiI\njRGxPSK279+/f7y1SpJq1NAPVDNzc2Z2ZWbX8uXLG3lpSZpRagn3vcCqEdsrK/vG20aS1CC1hPuj\nwJqIWB0Rc4BbgC2j2mwBPlq5a+adwKuZ+VKda5Uk1ai9WoPMHIiIO4AHgTbgnszcGRG3V45vArYC\nNwPdwAngtqkrWZJUTdVwB8jMrQwH+Mh9m0Z8n8An6luaJGmiYjiXm3DhiP3AC025eP0sAw40u4gW\nYn+czv54g31xusn0x1sys+odKU0L9xJExPbM7Gp2Ha3C/jid/fEG++J0jegP15aRpAIZ7pJUIMN9\ncjY3u4AWY3+czv54g31xuinvD+fcJalAjtwlqUCGew1qWM/+dyvr2O+IiEciYl0z6myEan0xot3b\nI2IgIj7YyPoarZb+iIgbI+LxiNgZEQ81usZGquH/lfMj4tsR8USlP4p94DEi7omIfRHx1BjHp/Y9\nGJnp11m+GH4q92fALwFzgCeAtaPaXAcsrny/HviXZtfdrL4Y0e77DD/49sFm193k341FwE+Bjsr2\nBc2uu8n98WngzyvfLwcOAXOaXfsU9ce7gKuBp8Y4fjPwABDAO+udG47cq6u6nn1mPpKZhyub2xhe\nOK1EtaztD/BJ4OvAvkYW1wS19MeHgfsy80WAzCy5T2rpjwQWRkQA5zIc7gONLbMxMvNhhv/7xjKl\n78Ew3Kuraa36ET7O8L/GJaraFxGxAvgAM+NtXLX8blwOLI6If4qIxyLiow2rrvFq6Y+/An4F6AF2\nAP81M4caU17LGW+2jEtNa8uoNhHxbobD/YZm19JEfwF8KjOHhgdnM147cA3wXmAe8OOI2JaZzzW3\nrKb5deBx4D3ApcD3IuKHmXm0uWWVx3Cvrqa16iPiSuBuYH1mHmxQbY1WS190AfdWgn0ZcHNEDGTm\nNxtTYkPV0h97gIOZ2Qv0RsTDwDqgxHCvpT9uAz6bw5PO3RHxc+AK4F8bU2JLmdL3YDgtU13V9ewj\nogO4D/hI4SOyqn2RmaszszMzO4GvAX9YaLBDbe86+BZwQ0S0R8R84B3A0w2us1Fq6Y8XGf4rhoi4\nEPhlYFdDq2wdU/oeDEfuVWRt69n/MbAU+GJlxDqQBS6SVGNfzBi19EdmPh0R3wGeBIaAuzPzjLfG\nTXc1/n58BvhyROxg+C6RT2VmkatFRsRXgBuBZRGxB/gTYDY05j0YPqEqSQVyWkaSCmS4S1KBDHdJ\nKpDhLkkFMtwlqUCGuyQVyHCXpAIZ7pJUoP8PNxiQ4BASF/gAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(precisions, recalls)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### scikit-learn中的Precision-Recall曲线" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.metrics import precision_recall_curve\n", + "\n", + "precisions, recalls, thresholds = precision_recall_curve(y_test, decision_scores)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(145,)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "precisions.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(145,)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "recalls.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(144,)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "thresholds.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VNXBx/HvyU5WIAQChBD2VUSJbCqCqCCtIq6odcEF\nUbHa7dXa9anW12oXfasVqSK4axWrIhVcQSsIAZQlgIQ9AULYsgBZ57x/3LGNEMgQZubOTH6f55kn\nmZmbmR/D5Mfhzr3nGGstIiISWaLcDiAiIv6nchcRiUAqdxGRCKRyFxGJQCp3EZEIpHIXEYlAKncR\nkQikchcRiUAqdxGRCBTj1hO3adPG5uTkuPX0IiJhadmyZXustRmNbedauefk5JCXl+fW04uIhCVj\nzFZfttNuGRGRCKRyFxGJQCp3EZEIpHIXEYlAKncRkQjUaLkbY2YYY3YbY1Yf435jjPk/Y0yBMWal\nMeZ0/8cUEZET4cvIfSYw9jj3Xwj08F4mA0+dfCwRETkZjR7nbq1daIzJOc4m44HnrbNe32JjTEtj\nTHtr7U4/Zfyu4nxY8xYMngzJjR7HLxIx1u8q572VO9yOIX6Qm9OaET0D21/+OImpI7C93vVC721H\nlbsxZjLO6J7s7OymPVvJOlj4CPS/VOUuzcrTCzYye0URxridRE7WlHO6hUW5+8xaOx2YDpCbm9u0\nlbm/fWdrYW9pZg5W19KzXTLzf3SO21EkDPjjaJkioFO961ne2wLk22GLyl2al8oaDwmx0W7HkDDh\nj3J/B7jee9TMUKA0YPvb4b8jd08t1B3jolG9RKDKmjoSYlTu4ptGd8sYY14BRgJtjDGFwG+AWABr\n7TRgLjAOKAAOAZMCFRaAqFjn69Mjjr1NRm+4YzHaOSmRpLLWQ2qCa3P9SZjx5WiZqxu53wJ3+i1R\nY7qeAxc8CDWVDd9fshZWv+l88Nq2T9BiiWzde5CaOkt8TBQJsdEkxEYRHxNNbLTB+GGgUVVTR4uU\neD8kleYg/IYBcUkw/K5j379/i1Pumz9TuUtQWGt5+P11PL1gU4P3p8THcNmgLNqnJZCSEEtyQgxt\nU+IZ0qX1CZV+ZU2d9rmLz8Kv3BvTsjOkdYItn8GQyW6nkQhX57H88p+reWXJNiae0Ynh3dtQWVNH\nVU0dlTUeKmvq+KxgD7MWbTnqo6CU+Bhy2iSRnZ5I59aJdGjZgpSEGO8lluR45/s2yfEkxEZ7P1DV\njCHim8grd2Mg52xY/x589Du307inywjoOtLtFBGtps7Dj1//mne/3sGdo7rx0wt6NTgSv2t0Dzwe\ny+GaOsoraymvrGHZ1v2s3lHK1r2HWF1Uyvurd1HnafhAgLiYKIZ2TWf/oWqN3MVnkVfuAP0ucc5i\n/ffjbidxh/XAF0/AXXnQsokni8lxVdbUccdLy/l43W7uu7A3U87pdtzto6IMSfExJMXHkJmWQI92\nKd+5v7bOw96D1ZRX1lJR5fwDUFFZS3llLet2lfPpN7upqvWQmZYQyD+WRBBjXTpsMDc312qZvQAp\nLYS/DoI+F8Nlf3c7TVgoq6zh0/UlzF+zi/2HqvnzlQNpl9pwkZZX1nDLrDyWbNnHg5f059ohnYOS\nsaS8ilaJscREa9dMc2aMWWatzW1su8gcuTd3aVkw7E747E8w9HboqIk6G1JcVskH+cXMzy9m0cY9\n1NT9d6Bz5sMfk52eSHbrRDq2bEGb5HjapMTTJimOpxZsJH9HGY9dNZDxAzsGLW+GjpSRE6Byj1Rn\n3gPLZsH8X8IPZp/gMf8GYuICFs1NBbsrmJ+/i/lrivlq+wEAOqcnMunMLozp146BnVqxbd8hZi8v\npGB3Bdv2HeLr7QfYf6jmP48RHxPF09cNYnSfdm79MUQapd0ykWzpM/DeT5r2sxc9DoNu9GscN3g8\nlq8LDzA/v5h5a3axqeQgAAOy0rigbzsu6JdJj7bJjR6SWFPnYf/BakoqqmidFEf7tBbBiC9yFO2W\nERg0CWISoKL4xH5uwaOwe11gMgWAtZZPvykhNSGGrFaJpLWI5cvN+5i/Zhcf5Bezu7yKmCjD0K7p\n3Dg8h/P6tKNDyxMr59joKNqmJtD2GPvhRUKNyj2SRUXDaT848Z9b8neorvB/ngBZ8E0Jk55betTt\niXHRjOyVwQV9MxnVqy1pibEupBNxh8pdjhabCDWH3E7hs0Wb9hIbbXjq2kHsLD3M7vIqTstuyfBu\nbXRcuDRbKnc5WlwS7FgBH/waEtNh8G0QG7q7I5Zs3seArJac11cfcIp8SwfMytGyzoCyHfDl007B\nf/yA24mO6VB1LasKSxncpbXbUURCispdjvb9P8Mvi53LoEmw6EnY9qXbqRq0fOsBaj1W5S5yBJW7\nHN/5v3NOinr7Tqg57Haao7y3agctYqMZnKNyF6lP5S7Hl5AKF/8V9m6Ajx+E6oMNXNz58PVwdR1z\nvt7JuFPakxSvj49E6tNvhDSu2yjv7pknnEtDRt4PI+8Naqx5a3ZRXlXL5YOygvq8IuFA5S6+Gfu/\nkHlKw8e/r5sLi5905rOJTw5apDeWFZLVqgVDtL9d5Cgqd/FNbAs44+aG78seBs+eD1+9HLQFUmrq\nPHyxcQ+3juhKVJTWyhU5kva5y8nrNNg5fHLxk+CpC8pT7i6vwmMhJz0pKM8nEm40chf/GDYV/nED\nvDUFkts2/XGiYmDIFEhtf9zNdpU6C6Rnaq4XkQap3MU/+lwEWYNh/dymP4a1UHMQUtrD0CnH3fTb\ncj/WghoizZ3KXfwjKhpu+eDkHqPmMPw+87jz2ng8lrdWFPGH99eREBtFVmtNvSvSEJW7hI4Y7yj8\nGCdLrdi2n9++m8/X2w9walYa064bRGqCZnoUaYjKXUJGZa2HKBNP2YFS2tS7fVdpJY+8v47ZK4rI\nSInnj1ecyqWnddRRMiLHoXKXkPHcv7dwuSeeNSsWM63kC/p3bElcTBQzv9hCbZ3ljpHduGNUd5J1\nNqpIo/RbIiFh/8Fq/vZpARV1F/KzmNfYUvYP/nf7BVTVehjbL5P7x/UhOz3R7ZgiYUPlLiHhrx8X\ncLCqlvF3/xEWVnHj2plcd/049rUfQUZKvNvxRMKOTmIS1y3buo8XFm/hytxO9MxMhUv+Bm37Ej37\nZjKqC92OJxKWNHIXV23de5Bbn19Gx5YtuHdsb+fGuCSY+BJMHwUvXQE9zm/8gRLSnBOpElIDG1gk\nTKjcxTWlh2qYNHMpHmt5btJgWiXF/ffOVjlw5Sx463b4+pXGH6yyDA5sgwnTApZXJJz4VO7GmLHA\n40A08Iy19uEj7k8DXgSyvY/5R2vtc37OKhHE47FMeXEZ2/cd4sWbh9ClTQNzxHQZAT9e49sDfvIQ\nLPgD9BoHfS/2b1iRMNToPndjTDTwJHAh0Be42hjT94jN7gTyrbWnAiOBPxlj4hA5hpVFpSzatJf7\nx/VhSNf0k3/AET+D9qfCnHugYvfJP55ImPNl5D4YKLDWbgIwxrwKjAfy621jgRRjjAGSgX1ArZ+z\nSgRZ+E0JxsDFp3bwzwNGx8KE6fD0CHj3bhj/ZNMfKzYRYjVnjYQ3X8q9I7C93vVCYMgR2zwBvAPs\nAFKAq6y1Hr8klIi08JsS+ndIIz3Zj4c5tu0N5/0G5t0Pj3Rp+uMkZcCP1zr/YIiEKX99oDoG+Ao4\nF+gGfGCM+cxaW1Z/I2PMZGAyQHZ2tp+eWsJJncfy+EcbyNu6n7tH9/D/Ewy53ZlV8mBJ035+80JY\nN8eZ30blLmHMl3IvAjrVu57lva2+ScDD1loLFBhjNgO9gSX1N7LWTgemA+Tm5tqmhpbwVFJexT2v\nreDfBXu57PQsppzTzf9PEhUF/S9t+s9b65S7R3sVJbz5Uu5LgR7GmC44pT4RuOaIbbYBo4HPjDHt\ngF7AJn8GlfD25aa93PXKCkoP1/DIZQO48oxOjf+QG6K9vxJ11e7mEDlJjZa7tbbWGDMVmIdzKOQM\na+0aY8wU7/3TgAeAmcaYVYAB7rXW7glgbgkjBbvLueaZL8luncismwbTp30In2gU5d0VU1fjbg6R\nk+TTPndr7Vxg7hG3Tav3/Q7gAv9Gk0ixeNM+6jyWWZMGh/7kX3He4+2rK9zNIXKSNLeMBFz+zjLS\nWsTSKRxWTUrxrt1avtPdHCInSeUuAbemqJQ+7VNwToMIcSmZztfyXe7mEDlJKncJqPLKGlbvKGNQ\n51ZuR/GNRu4SIVTuElBLtzj724d3a9P4xqEgLtGZYbL0yKN9RcKLyl0CpnD/IZ76dCNxMVHhM3IH\nSO8Be75xO4XISdGUv+J3NXUenv18M49/uAGA313cj4TYaJdTnYC2fWD9v9xOIXJSVO7iV8u27uMX\nb61m3a5yzu/bjt9e3I+OLcPgKJn62vaFFS9ARQkkZ7idRqRJVO7iN+t3lXPFtEVkpiYw/bpBXNAv\n0+1ITdO2j/N1dz4kn+NuFpEmUrnLSVu+bT8ZyfF8tK4Yj4W37jyTdqlhPGVuh4HOtL95M6Cryl3C\nk8pdTspHa4u55fk8oo0hrUUsvTNTwrvYAVq0gjPvgU8fgq2LoPMwtxOJnDAdLSNNtnZnGT98ZQX9\nO6RxzZBsyitrOb9vO7dj+cfwqZDSAeb9HDxamkDCj0bu0iQl5VXcMiuPlIRYnrkhl3apCdw7tjfx\nMREyXohLchb+eOs2WPU6nDrR7UQiJyRCfhMlmCpr6pj8Qh77Dlb/p9gBkuJjiImOoLfUKVdCh9Ng\n/i9h/xa304ickAj6TZRgsNbyszdWsmLbAf5y1UD6d0xzO1LgREXBhKedud1fvAwO7nU7kYjPVO5y\nQr7YuJd3v97Bz8b0Ymz/MD3U8URk9IKrX4MD2+GVq6D6kNuJRHyicpcTsmnPQQAuH5TlcpIg6jwM\nLnsGCvPgjZugTkvwSejTB6pyQnYeOExMlKFNcrzbUYKr78Uw7lGY+1N4cQK07Hz0NrGJzlE2LbX4\nu7hP5S4nZFVRKd0ykomOCoO52f1t8K1QVQ5Ln4E9BUfff2gvrHzN2U/fa2zw84nUo3IXn9XUeVi2\ndX/z2iVzpLN/7Fwasncj/OMGZ9/88Ltg9G8gOja4+US8tM9djslay+Y9B/F4LAAf5hdzqLqOM7uH\nydzswZbeDW7+EHJvhi/+Cs+Ncz6IFXGBRu5yTPPWFDPlxWV0SEtgwukd+SC/mK4ZSZzXJ0LOQg2E\n2AT4/p8h50x45254+my4ZJp200jQaeQux/SPvO20SY6nZ2YKT326kW+KK7h7dI/mub/9RPW/DG5b\nAGlZ8MpEZ5eNSBBp5C4N2lVayYJvSrj57C78/MI+7C6rZPWOUkb1aut2tPCR3g3G/RFmjIH9m53r\nIkGicpcGTVvgjDR/MMQ55K9tagLnhvtsj26IT3W+Vpa5m0OaHe2WkaPsqaji5SXbuHxQFp1aJ7od\nJ7wleKdnqFK5S3Bp5C7fsW3vIUY8+gkA1w/LcTdMJEjQyF3coZG7/MemkgqufHrRf673aZ/iYpoI\nEZcMJkojdwk6jdwFcI5pv37GEmrqPPxwdA/6dUjFGB0Vc9KMgfgUqCx1O4k0Myp3AaDscC2F+w/z\ny+/14Zazu7odJ7LEp2m3jASdyr2ZKz1cw7zVu5i9ohCAjJRmNiFYMCSkareMBJ3KvRmqrfMwP7+Y\nt78q4pN1JVTXechJT+SHo3tEzhqooSQxHXZ8BWU7IbW922mkmfCp3I0xY4HHgWjgGWvtww1sMxJ4\nDIgF9lhrz/FjTvGjV5Zs41dvryEjJZ4fDO3M+IEdGJCVpn3sgXLur+CFS2DWRXDjHEhpBouciOsa\nLXdjTDTwJHA+UAgsNca8Y63Nr7dNS+BvwFhr7TZjjE5jDGF7KqoBWHTfuZG15mmo6nQGXPuGs1Tf\nrIvgxvcgWb8iEli+/GYPBgqstZustdXAq8D4I7a5Bphtrd0GYK3d7d+Y4k8bSypolRirYg+mzsPg\n2n9AaaFT8BUlbieSCOfLb3dHoP68pYXe2+rrCbQyxnxqjFlmjLneXwHFf6pq6/jXqp3MX1PM+IFH\n/hVKwOWcCde8Dvu3OgVfc9jtRBLB/PWBagwwCBgNtAAWGWMWW2u/qb+RMWYyMBkgO1tLkQWDtZbl\n2/bz5vIi3lu5k9LDNWSmJnDj8By3ozVPXc6Gix6HtyY7H7J2HuZ2IolQvpR7EdCp3vUs7231FQJ7\nrbUHgYPGmIXAqcB3yt1aOx2YDpCbm2ubGlqOr2B3BYX7DzGiRwa3PJ/Hx+t2kxAbxZh+mUw4rSNn\ndW+jXTJuyujpfD28z90cEtF8KfelQA9jTBecUp+Is4+9vreBJ4wxMUAcMAT4iz+Dim9KD9fwg2e+\nZFdZJd0ykthYcpArc7P49UX9SI7Xka8hoUVr5+shlbsETqPDN2ttLTAVmAesBV631q4xxkwxxkzx\nbrMWeB9YCSzBOVxydeBiy7H87t18SiqquO/C3sTHRAMw7pT2KvZQ0qKV8/XwfndzSETz6TfeWjsX\nmHvEbdOOuP4o8Kj/osmJ+iC/mDeXF3LXud2Zck43bhvRle37DtOpdQu3o0l98SkQkwC717qdRCKY\ndrxGCGstD76XT+/MFO46twcAxhiy0xN1clKoMQZOvwFWvgrF+Y1vL9IEKvcIsWjjXrbuPcTNZ3Uh\nLkZ/rSFv5H3OKk3z7gerYwvE/9QCEWDdrjLueHk5HVu2YGx/ndoeFhJbOwW/6RPYMN/tNBKBVO4R\n4P7Zq4iLjuKVW4eSkhDrdhzx1Rm3QHp3mPcLqKt1O41EGJV7mNtYUsGqolIuPT2L7HStdxpWomNh\n9K9h7wbYMM/tNBJhVO5hbM7KHVz8189Jjo9hwmmaTiAs9foepHSAvBluJ5EIo3IPUy9/uY2pL6+g\nd/tU5t59Nr0ytd5pWIqOgUE3QMFHsG+z22kkgqjcw9An63dz/1urOLd3W16+dQjt03Qce1g7/Xpn\nEe1lM91OIhFE5R5missqeXBOPl3aJPHUD07/z1moEsZSO0CvC2HJdFj1httpJEKo3MPI63nbOe9P\nCyjcf5jfXNRXxR5Jxv0RMgfAmzfDnB9BTaXbiSTMqdzDxL6D1dz75kp6ZqYw754RjOyllXwiSmp7\nZwm+4T90Plx99nzYu9HtVBLGVO4hrrrWw/urd3LPa19hLVw3tDM5bZLcjiWBEB0LFzwAV78KB7bB\n9JGQ/7bbqSRMqdxD2KrCUoY//BFTXlzO+l1lTB3VnfP7tnM7lgRarwvhtoXQpge8fj188pDbiSQM\naR7YEFVT5+GBOc6kUjNuzGVEjwwtsNGctOoMk9539sF//hcYNhUSUt1OJWFEbRFirLWs3VnGjc8t\nYcmWffzkgl6c27udir05iomDoXdAXbXmn5ETppF7iMjfUcbbXxXx/ppdbN17iNhow6OXD+CK3E6N\n/7BErk6DISkD1s2BUy53O42EEZV7CPggv5jbX1wGwPDubbhtRDfO79uOjJR4l5OJ66Kioff3nOPf\nayohNsHtRBImVO4uqvNYpi3YyGMffkO/jmnMvPEMWiXFuR1LQk3vi5yzVzcvhJ4XuJ1GwoR25Lrk\nUHUtk2Yu5dF567mgbybP3zRYxS4NyzkLYlpAwYduJ5EwopG7S6a+vILPN5Tw0IRTuHpwJy2FJ8cW\nm+AUvMpdToBG7i74fMMePl63m3vO68k1Q7JV7NK47ufBvo2aOVJ8pnIPsiWb9zFp5hJ6tE3muqGd\n3Y4j4aL7ec7X16+HPRvczSJhQeUeZH//bBMpCbG8cftw7WMX36V3g1OugJJ18EQuvHQlbPpUi2vL\nMancg+iFxVv5IL+YG4blkNZCa53KCTAGLnsGfpQPI38OO5bD8+Nh2lmw4iWorXI7oYQYlXuQzFm5\ng1/9czXn9m7L1HO7ux1HwlVyBoy8D+5ZDRc/AdYDb98Bf+kPCx6Bg3vcTighwliX/luXm5tr8/Ly\nXHnuYCvcf4gxf1lI7/apvHTLEBJiNQ+7+Im1zu6ZRU9CwQcQkwADroRhd0FGT7fTSQAYY5ZZa3Mb\n206HQgZYda2Hn89eRZ21PD5xoIpd/MsY6DbKuZSsh8V/g69fhVVvwj0rIamN2wnFJdotE2DTFmzk\nsw17+O1F/chqleh2HIlkGb3gosfh1o+h5qDWZG3mVO4BtHZnGa/nbad722QmDs52O440F+36QZdz\nnBWd6mrdTiMuUbkHyNIt+7hi2iKqaz08NOEUt+NIczPkNigrgvVz3U4iLlG5+1npoRreWlHITTOX\n0jY1nnemnsXgLq3djiXNTc+xkJYNS6a7nURcog9U/WRjSQW/ezeffxfsodZj6ZyeyPM3DSYzTVO0\niguioqH/pbDoCaitdhb+kGbFp3I3xowFHgeigWestQ8fY7szgEXARGvtG35LGYIqqmrJ27KP5Vv3\ns3zbAZZs2UdiXDS3jujK+X3bMTCrJVFRmjNGXJR5CnhqYc83kNnf7TQSZI2WuzEmGngSOB8oBJYa\nY96x1uY3sN0fgIhfD+yjtcXc++ZK9lRUE2Wgd2Yq1wzO5o6R3WibqpG6hIi2fZ2vu/NV7s2QLyP3\nwUCBtXYTgDHmVWA8kH/EdncBbwJn+DVhiDhcXceSLft49+sdvLGskN6ZKfz5yoGc3rkVyfHauyUh\nKL07RMVC8Rq3k4gLfGmljsD2etcLgSH1NzDGdAQmAKOIwHJ//MMNPPlJAdV1HuKio7j17C78dEwv\n4mN0QpKEsJg4aNcXVr7mTDqm0Xuz4q8h52PAvdZaz/HmJjfGTAYmA2Rnh/5x37vLKnnsow28/OU2\nerVL4efjejOkSzot4lTqEiYufgJevhJmjIUrnoMe57udSILEl0Mhi4BO9a5neW+rLxd41RizBbgc\n+Jsx5pIjH8haO91am2utzc3IyGhi5OAoKa/i5ll5vLGskKsHd+KVyUMZ2autil3CS/sBzhmrrbs4\nJb/k724nkiDxZeS+FOhhjOmCU+oTgWvqb2Ct7fLt98aYmcAca+0//Zgz4Cqqapm/Zheri8pYvaOU\nVYWl1Hks/3f1aYztn+l2PJGmS+0Ak/4Fb94Cc38KezfCmN87h0tKxGq03K21tcaYqcA8nEMhZ1hr\n1xhjpnjvnxbgjEHxw1dW8PG63STERtG3fSpX5mZx3bAcurdNdjuayMmLT4aJL8H8X8HiJ6F0O1z5\nAkTpPMZI5dM+d2vtXGDuEbc1WOrW2htPPlZwHThUzafrd/P9Ae157KqBxETrDS8RKCoaxj7kjOTn\n/8KZQXL4VLdTSYA0+xaz1jL15RXEREVx81ldVOwS+YbdCb2+Bx/9DnavdTuNBEizbrIvNu7hpplL\n+bxgD1PO6cpp2a3cjiQSeMY4UwPHp8Dsyc70BBJxmmW5V9bUMeuLLVz7zJcs3bKf64Z25obhOW7H\nEgme5Ayn4HethIWPuJ1GAqDZnFpZW+ehoKSC1UVlzPxiM6uLyhiQlcbT1w2ifVoLt+OJBF+f78PA\na+GzP0G7/tDvqKOXJYw1i3JfvGkvd72ygpJyZ4X4xLhoHppwClcP7sTxTroSiXjjHoW9Bc5hkvHJ\n0P08txOJn0RcuR+sqmVXWSWZqQnMXl7I14WlvLGskLYp8Tx21UD6d0yjS5skojVjowjEJcE1r8PM\n78Nr18F1/4TsIY3/nIS8iCv3KS8u47MNe/5zPT0pjlG9Mrh1RFeGd9NiwSJHadESrpvtTFHw0hUw\n6T1numAJaxFV7h/kF3+n2O8e3YN7zuuhXS8ijUluC9e/DTPGwAsT4PYvnNskbEVEudd5LPPX7OL2\nl5bTpU0Sr982jINVteS0SXI7mkj4aNkJLp/hFPyWz52VnCRshXW5l1fW8MCcfP61ehfllbV0SEvg\nxVuGkJEST0ZKvNvxRMJPO++0wPs3u5tDTlrYlvuBQ9VcMW0Rm/Yc5LLTO3JWjwzO79NOszaKnIz4\nZEhqC/tU7uEu7Mp93a4ynl6wiTkrd1DnsTx74xmM6qV9gyJ+0yoH9m9xO4WcpLA7Q/XzDXt4a0UR\nNXWWxyaepmIX8bd2fWHbIvjoAag57HYaaaKwK/cRPTNITYjhhZsHc/GpHdyOIxJ5Rv8G+l8On/0R\nnhoOGz9xO5E0gbHWuvLEubm5Ni8vz5XnFhEfbPoU5vwI9m2CAVfBmIcgSeeKuM0Ys8xam9vYdmE3\ncheRIOk6Em5fBCP+B1bPhidyYfnz4PG4nUx8oHIXkWOLTYBzfwG3/xsy+sA7d8Gsi+DgnsZ/Vlyl\ncheRxmX0ghvfg4v/CkV5zlQFpYVup5LjULmLiG+iouD06+EHs6GiGJ4dAyXfuJ1KjkHlLiInJudM\nZxRfVwXPjYWi5W4nkgao3EXkxLUfADfNg9gkZx/8pgVuJ5IjqNxFpGnSu8HN8yCtE7x0Oax83e1E\nUo/KXUSaLrUD3PQvyBoMs2+FBY+CS+fOyHep3EXk5LRo5Sz2MeAq+ORBeHsq1NW4narZC7uJw0Qk\nBMXEw4SnnUnHFvwBSrfDlc87qzyJKzRyFxH/MAZG3Q+XPAVb/+0cC3/4gNupmi2Vu4j418Br4Np/\nwN4Nzhmt2gfvCpW7iPhft3Nh9K9h7TuQ96zbaZollbuIBMawu6D7+fD+/bBrldtpmh2Vu4gERlSU\ns/+9RSv4xySoqnA7UbOicheRwEnOgEunw94CmPszt9M0Kz6VuzFmrDFmvTGmwBhzXwP3X2uMWWmM\nWWWM+cIYc6r/o4pIWOp6Doz4GXz9Mnz9qttpmo1Gy90YEw08CVwI9AWuNsb0PWKzzcA51tpTgAeA\n6f4OKiJh7Jx7IXs4zPkx7ClwO02z4MvIfTBQYK3dZK2tBl4FxtffwFr7hbV2v/fqYiDLvzFFJKxF\nx8BlzzgnO71xI9RUup0o4vlS7h2B7fWuF3pvO5abgX+dTCgRiUBpHZ0PWHetgk8fcjtNxPPr9APG\nmFE45X7WMe6fDEwGyM7O9udTi0g46DUWcs6GLZ+7nSTi+TJyLwI61bue5b3tO4wxA4BngPHW2r0N\nPZC1drq1Ntdam5uRkdGUvCIS7lLaaw3WIPCl3JcCPYwxXYwxccBE4J36GxhjsoHZwHXWWq27JSLH\nlpgOh/YhvYh7AAAIs0lEQVS5nSLiNbpbxlpba4yZCswDooEZ1to1xpgp3vunAb8G0oG/GWMAaq21\nuYGLLSJhKzEdqsuhtsr5gFUCwqd97tbaucDcI26bVu/7W4Bb/BtNRCJSclvna1kRtO7qbpYIpjNU\nRSS4OpzmfC1c5m6OCKdyF5HgatsX4pJh+5duJ4loKncRCa7oGMjKhe2L3U4S0VTuIhJ8nYZA8Rr4\n+PewbTHU1bqdKOJoDVURCb7TroPNC+GzP8LCRyA+DbqOcBb56DYaWnV2O2HYU7mLSPC17AQ3vQ+H\n98OmBbDxY+ey9l3n/vTu/y36nLMgPtndvGHIWJfWN8zNzbV5eXmuPLeIhCBrYc8G2PgRFHzkTFFQ\nexiiYiF7qFP23UdDu1OchUCaKWPMMl/OI1K5i0hoqq2CbYucot/4MRSvdm5PyvAW/XnQbwJEx7qb\nM8h8LXftlhGR0BQTD11HOhcegPJdsPET78j+Q1j5mjPaP/UqV2OGKpW7iISHlEwYeLVzqTkMv8+E\nskK3U4Ws5rvjSkTCV2wL5wibit1uJwlZKncRCU/JbaGi2O0UIUvlLiLhKbkdHNje+HbNlMpdRMJT\n15FQlAebP3M7SUhSuYtIeBo+FVpmw7/+R9MXNEDlLiLhKbYFjHkIdufD0mfcThNyVO4iEr56fx+6\njoJPHoKKErfThBSVu4iEL2Pgwkeg5iAsfNTtNCFF5S4i4S2jJwyYCMtn6bj3elTuIhL+zvoR1FXD\noifcThIyVO4iEv7adId+l8LSZ+HQPrfThATNLSMikeHsn8DqN2DWRZB5inOYZFonZ+74ltmQmgUx\ncW6nDBqVu4hEhnZ9Ycz/wtp3nBObyneA9dTbwDiTj6V5y75lJ+/3nf/7fVyia/H9TfO5i0hkqquB\nsiJnioID26B0u/f7rc73pUXgqfnuzySm1xvxZ3939J/WCVq0dOfPUo/mcxeR5i06FlrlOJeGeOqc\nOeKPLP0D26FkHWyYD7WV3/2Z+LR6I/76o3/v/wAS053DM0OAyl1EmqeoaEjr6Fyyhx59v7VwcA+U\nbnNG/ge2e8vfe33L51Bd/t2fiU2EtKzvjvhbd4Ve45zFR4JI5S4i0hBjIDnDuXQcdPT91kLlgSNK\nf7v3H4PtULQcDnuP3OkxBia+FNQlAVXuIiJNYQy0aOVc2g9oeJuqCljxIrx/L7x9J1wyLWiLe6vc\nRUQCJT4Zhk5xdt98/KDzD8HYh4OyX17lLiISaGf/FA7th8VPQovWMPLegD+lyl1EJNCMgQsehMP7\n4dOHILE1DL41oE+pchcRCYaoKLj4r84cOC2zA/90vmxkjBlrjFlvjCkwxtzXwP3GGPN/3vtXGmNO\n939UEZEwFx0Dlz8LPccE/KkaLXdjTDTwJHAh0Be42hjT94jNLgR6eC+Tgaf8nFNERE6ALyP3wUCB\ntXaTtbYaeBUYf8Q244HnrWMx0NIY097PWUVExEe+lHtHYHu964Xe2050G4wxk40xecaYvJISLYkl\nIhIoQZ3P3Vo73Vqba63NzcjICOZTi4g0K76UexHQqd71LO9tJ7qNiIgEiS/lvhToYYzpYoyJAyYC\n7xyxzTvA9d6jZoYCpdbanX7OKiIiPmr0OHdrba0xZiowD4gGZlhr1xhjpnjvnwbMBcYBBcAhYFLg\nIouISGN8OonJWjsXp8Dr3zat3vcWuNO/0UREpKlcW4nJGFMCbHXlyR1tgD0uPr8vlNE/wiEjhEdO\nZfSfpubsbK1t9IgU18rdbcaYPF+WqnKTMvpHOGSE8MipjP4T6JxBPRRSRESCQ+UuIhKBmnO5T3c7\ngA+U0T/CISOER05l9J+A5my2+9xFRCJZcx65i4hErGZV7saYK4wxa4wxHmNMbr3bc4wxh40xX3kv\n0473OG5k9N73c++c+euNMYGfENpHxpjfGmOK6r1+49zO9K3G1iIIBcaYLcaYVd7XLs/tPN8yxsww\nxuw2xqyud1trY8wHxpgN3q+tQjBjSL0fjTGdjDGfGGPyvb/bd3tvD+hr2azKHVgNXAosbOC+jdba\ngd7LlCDnqq/BjN459CcC/YCxwN+8c+2Hir/Ue/3mNr554Pm4FkGoGOV97ULpEL6ZOO+1+u4DPrLW\n9gA+8l5300yOzgih9X6sBX5ire0LDAXu9L4PA/paNqtyt9autdaudzvH8Rwn43jgVWttlbV2M85U\nD4ODmy7s+LIWgRyDtXYhsO+Im8cDs7zfzwIuCWqoIxwjY0ix1u601i73fl8OrMWZEj2gr2WzKvdG\ndPH+F26BMeZst8M0wKc58110l3eJxRlu/1e9nlB/zb5lgQ+NMcuMMZPdDtOIdvUmBdwFtHMzzHGE\n4vsRY0wOcBrwJQF+LSOu3I0xHxpjVjdwOd6IbSeQba0dCPwYeNkYkxpiGV3VSOangK7AQJzX8k+u\nhg0/Z3nfexfi/Jd9hNuBfOGdUyoUD7cLyfejMSYZeBO4x1pbVv++QLyWPk0cFk6stec14WeqgCrv\n98uMMRuBnkBAPtxqSkZcnjPf18zGmL8DcwIcx1dhsc6AtbbI+3W3MeYtnN1JDX0uFAqKjTHtrbU7\nvUtp7nY70JGstcXffh8q70djTCxOsb9krZ3tvTmgr2XEjdybwhiT8e2Hk8aYrjgLfW9yN9VR3gEm\nGmPijTFdcDIucTkTAOa76+VOwPlQOBT4shaBq4wxScaYlG+/By4gdF6/hrwD3OD9/gbgbRezNCjU\n3o/GGAM8C6y11v653l2BfS2ttc3mgvMXXYgzSi8G5nlvvwxYA3wFLAcuCrWM3vt+AWwE1gMXuv16\n1sv1ArAKWOl9w7Z3O1O9bOOAb7yv2y/cztNAvq7A197LmlDKCLyCs1ujxvuevBlIxzmyYwPwIdA6\nBDOG1PsROAtnl8tKb8d85X1fBvS11BmqIiIRSLtlREQikMpdRCQCqdxFRCKQyl1EJAKp3EVEIpDK\nXUQkAqncRUQikMpdRCQC/T/h2z9YI1kP+wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(thresholds, precisions[:-1])\n", + "plt.plot(thresholds, recalls[:-1])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFcNJREFUeJzt3X+Q1fV97/Hnm4UV+aXILqAgAgYkaGOrq+Zap0lsEn9k\nrE2TSTWd2pqk1Dsxt9PepObemTTNtX+Yye1MbxIjZRxjk94Jd9rYBBOi7a1N7NRrFVMRUHFWEFl+\nyK4QfiMs+75/7NEcN8B+D5yz5+yX52OGmT3f7+ec78uvy4vv+Xy/53wjM5EklcuYZgeQJNWf5S5J\nJWS5S1IJWe6SVEKWuySVkOUuSSVkuUtSCVnuklRClrskldDYZm24o6Mj586d26zNS9Ko9Mwzz/Rl\nZudw45pW7nPnzmXVqlXN2rwkjUoRsanIOKdlJKmELHdJKiHLXZJKyHKXpBKy3CWphIYt94h4ICJ2\nRMTa46yPiPhqRHRHxHMRcVn9Y0qSalHkyP1B4PoTrL8BWFD5swS479RjSZJOxbDlnpmPAztPMORm\n4Fs56Eng7Ig4t14Bj+fg4aP8+Yp1bN55oNGbkqRRpx5z7rOAzVWPeyrLfkFELImIVRGxqre395Q2\nuvzpV3nwiVfY+rODp/Q6klRGI3pCNTOXZWZXZnZ1dg776dnjOtw/wLLHN3DF3KlcNX9aHRNKUjnU\no9y3AOdXPZ5dWdYwD/20h227D/Hp972jkZuRpFGrHt8tswK4MyKWA1cBuzNzWx1e95j6jw5w309e\nZn7nRBbMmFzztEzHpDNoH+sVoJLKbdhyj4jvAO8FOiKiB/giMA4gM5cCK4EbgW7gAHB7o8ICrO7Z\nzabXB0+i/uo9j9X8/Pe/czr3/94V9Y4lSS1l2HLPzFuHWZ/Ap+uWaBi/NOssvnbrr3DgcH9Nz3th\n214efOIVLpl1VoOSSVLraNpX/p6s9rFjuOnS82p+3u3ffIqzJ4zjE9fMa0AqSWotp8Xk8zObdvIv\n63v5w1+7kCnjxzU7jiQ13Kg7cj8ZX3usG4DHXnyNJ17ua3Ia1eraRdO5/Vd9xyXV4rQo90vOO4vd\nB4/QP5Dse6O2uXo1z95D/XTv2MeUM8dZ7lKNToty/+x1F/FZLmp2DNXovz20hlf69vMnH1jY7CjS\nqHNazLlr9Hl+6x7+z9Ovctt/msuFnZOaHUcadU6LI3eNLpnJ3T94ngQWzJjEyjUN+0zcW94xfRIL\nZ0xu+HakkWK5q+UcOjLAqk07yRycmhkJF0ybwE8+974R2ZY0Eix3tZwz29v4t7uuZdeBIw3f1lOv\n7OQL31vLx7rOH36wNIpY7mpJ06eMZ/qU8Q3dxsBA8qd/v5qZU8bzCa/GUcl4QlWnrR+s2cbqnt18\n9rqLOLO9rdlxpLryyF2nrft+/DIAX1qxjv/x8LrCzxs/ro1v3n4FF5/n9xSpdVnuOm196pp5rNmy\nu6bnPLJ2O3373mDyGX6NhVqb5a7T1kcun81HLp9deHz3jn18+8lN/O67L2DOtAkNTCadOufcpYK+\n/MiLnDmujc9c6x3A1Po8cpcKWPXKTv7p+ddYMH0Sf/34hoZv77cum8WimVMavh2Vl+UuFbDn0BEm\nnzGWnl0H+fb/29Sw7RwdSA4fHWBi+1jLXafEcpcKuHbRDNZ86bqGb+dzf7ea76/eyi1X+qEqnRrn\n3KUW8Urffh76jy38zlVzmNHgD3Cp/Dxyl1rE1x7rJjO57uKZbOjd1+w4dTd32kTGjIlmxzhtWO5S\nC8hMvv/sFgYSbln2ZLPjNMRnrn0H//WD3ldhpFjuUguICJYveTdbfnaw2VHq7vmte/jrxzcwr2Ni\ns6OcVix3qUV0zT2HrmaHaICfbtrFGWPH8IHFM5od5bTiCVVJDdN/dIAfrtnGtYumM3m8X9kwkix3\nSQ3z5Iad9O07zG9cel6zo5x2LHdJDfPw6q1MOmMs71s0vdlRTjuWu6SGeKP/KD9au40PLp7B+HF+\nX/5Is9wlNcS/vtTHnkP93PTLTsk0g1fLSGqIFau30jYm6N3zBg/9tKfZcQAYE8F7FnYydWJ7s6M0\nnOUuqSE27TzA0YHkT7/7XLOjvM0fv38hf/T+Bc2O0XCFyj0irgf+F9AG3J+Z9wxZfxbwt8Ccymv+\nz8z8Zp2zShpFvvMHV9G7941mx3hL9459fPJvVjFn2pnNjjIihi33iGgD7gU+APQAT0fEisx8vmrY\np4HnM/OmiOgE1kfE/87Mww1JLanlTWgfywXTWmdy4JlNuwC45DS5922RE6pXAt2ZuaFS1suBm4eM\nSWByRAQwCdgJ9Nc1qSSdgrVb9jB+3Bjmd05qdpQRUeSf1VnA5qrHPcBVQ8Z8HVgBbAUmA7+dmQN1\nSShJddDdu49DRwZ45xceGXZs+9gxPHj7FXTNPWcEkjVGvd4zXQc8C1wLXAj8U0T8a2buqR4UEUuA\nJQBz5syp06YlaXh3vGc+F583/N2tenYd5OHVW9l7aHRPPhQp9y1A9W1hZleWVbsduCczE+iOiI3A\nIuCp6kGZuQxYBtDV1ZUnG1qSanX1hR1cfWHHsOP+btVmHl69ddR/i2WROfengQURMS8i2oFbGJyC\nqfYq8OsAETEDuAho/F2EJanONvTtZ1xbMHvq6L6qZtgj98zsj4g7gUcZvBTygcxcFxF3VNYvBe4G\nHoyINUAAd2VmXwNzS1JDbOjdx5xzJjC2bXR/gL/QnHtmrgRWDlm2tOrnrcAH6xtNkkbexr79pbii\nZnT/0yRJdXR0IHnl9QPMH+Xz7WC5S9Jbtv7sIIf7B5jfablLUmm83LsPgHkdTstIUmls7NsP4JG7\nJJXJht79TB4/lmkl+Epgy12SKjb07WN+5yQGvyZrdLPcJaliY+/+UlwpA5a7JAFw8PBRtu4+ZLlL\nUpl07xi8UubC6aP/Shmw3CUJgPWv7QXgopmTm5ykPix3SQLWb99D+9gxzJ3mtIwklcb61/axYPok\n2saM/itlwHKXJABe2r6Xi2aUY0oGLHdJYveBI2zfc6g08+1guUvSWydTF5ao3Ot1D1VJGrXeLPcF\n0ycxMFD8DqARtOynWS13Sae9TZUvDLvmy/9S0/M+ctls/vJjlzYi0imz3CWd9m69ag5TzhxHFj9o\n59tPvkLfvjcaF+oUWe6STnsXdk7iv/z6gpqe87f/volzzxrfoESnzhOqklSjw/0D9O17gxlTLHdJ\nKo0dew+RiUfuklQmr+05BMAMy12SymPb7sFy98hdkkpk+5vlPuXMJic5Pstdkmq0ffchxo8bw5Qz\nW/eCQ8tdkmq0fc8hzj3rzJb9dCpY7pJUs+27DzFjyhnNjnFClrsk1ejNI/dWZrlLUg0GBpLX9hxi\nZgtfKQOWuyTV5PX9hzlyNJnZwp9OBctdkmqy5WcHATjv7BJMy0TE9RGxPiK6I+Lzxxnz3oh4NiLW\nRcRP6htTklpDz64DAJx/TmuX+7AXaUZEG3Av8AGgB3g6IlZk5vNVY84GvgFcn5mvRsT0RgWWpGba\nvHPwyH321AlNTnJiRY7crwS6M3NDZh4GlgM3DxnzceChzHwVIDN31DemJLWGzbsOMHXCOCad0bof\nYIJi5T4L2Fz1uKeyrNpCYGpE/DginomI2471QhGxJCJWRcSq3t7ek0ssSU20eecBzj+ntY/aoX4n\nVMcClwMfAq4DvhARC4cOysxlmdmVmV2dnZ112rQkjZwtuw5yfotPyUCxct8CnF/1eHZlWbUe4NHM\n3J+ZfcDjQGveWFCSTtLAQNKz6yCzW/xkKhQr96eBBRExLyLagVuAFUPGfB+4JiLGRsQE4CrghfpG\nlaTm2rH3DQ4fHWj5k6lQ4GqZzOyPiDuBR4E24IHMXBcRd1TWL83MFyLiEeA5YAC4PzPXNjK4JI20\nzW9eBjm19Y/cC53uzcyVwMohy5YOefwV4Cv1iyZJrWVj334A5nVMbHKS4fkJVUkqaGPffsa1BbNa\n/NOpYLlLUmEbe/cz55wJjG1r/eps/YSS1CI29u1nXsekZscoxHKXpAIGBpJXXt/P/M7Wn28Hy12S\nCtm25xBv9A8wd5rlLkmlsbF39FwpA5a7JBWysW8fgNMyklQmL/fuZ2J7G9Mnt/aNsd9kuUtSAeu3\n72XBjMlERLOjFGK5S9IwMpP1r+3lohmTmx2lMMtdkobRt+8wO/cf5qKZlrsklcb67XsBLHdJKpP1\nr1nuklQ667fvYdrEdjomjY4rZcByl6Rhrd++d1QdtYPlLkknNDCQvPTaPstdkspk084DHDxylEWW\nuySVx5otuwG4+LyzmpykNpa7JJ3Aui27aW8bw8JR9AEmKHgPVUk6Xa3Zspv5nRPZe+hIXV5vTART\nJ7bX5bVOxHKXpBPY2LefbbsPcflf/N+6veaXfuNifu/quXV7vWOx3CXpBP7yY5fSvWNf3V7vSw8/\nz7bdh+r2esdjuUvSCVx9YQdXX9hRt9f7ix++ULfXOhFPqEpSCVnuklRClrsklZDlLkklZLlLUglZ\n7pJUQpa7JJVQoXKPiOsjYn1EdEfE508w7oqI6I+Ij9YvoiSpVsOWe0S0AfcCNwCLgVsjYvFxxn0Z\n+Md6h5Qk1abIkfuVQHdmbsjMw8By4OZjjPsM8F1gRx3zSZJOQpFynwVsrnrcU1n2loiYBXwYuK9+\n0SRJJ6teJ1T/CrgrMwdONCgilkTEqohY1dvbW6dNS5KGKvLFYVuA86sez64sq9YFLI8IgA7gxojo\nz8zvVQ/KzGXAMoCurq482dCSpBMrUu5PAwsiYh6DpX4L8PHqAZk5782fI+JB4AdDi12SNHKGLffM\n7I+IO4FHgTbggcxcFxF3VNYvbXBGSVKNCn2fe2auBFYOWXbMUs/M3z/1WJKkU+EnVCWphCx3SSoh\ny12SSshyl6QSstwlqYQsd0kqIctdkkrIcpekErLcJamELHdJKiHLXZJKyHKXpBKy3CWphCx3SSoh\ny12SSshyl6QSstwlqYQsd0kqIctdkkrIcpekErLcJamELHdJKiHLXZJKyHKXpBKy3CWphCx3SSoh\ny12SSshyl6QSstwlqYQsd0kqIctdkkqoULlHxPURsT4iuiPi88dY/zsR8VxErImIJyLi0vpHlSQV\nNWy5R0QbcC9wA7AYuDUiFg8ZthF4T2b+EnA3sKzeQSVJxRU5cr8S6M7MDZl5GFgO3Fw9IDOfyMxd\nlYdPArPrG1OSVIsi5T4L2Fz1uKey7Hg+CfzoWCsiYklErIqIVb29vcVTSpJqUtcTqhHxPgbL/a5j\nrc/MZZnZlZldnZ2d9dy0JKnK2AJjtgDnVz2eXVn2NhHxLuB+4IbMfL0+8SRJJ6PIkfvTwIKImBcR\n7cAtwIrqARExB3gI+N3MfKn+MSVJtRj2yD0z+yPiTuBRoA14IDPXRcQdlfVLgT8DpgHfiAiA/szs\nalxsSdKJFJmWITNXAiuHLFta9fOngE/VN5ok6WT5CVVJKiHLXZJKyHKXpBKy3CWphCx3SSohy12S\nSshyl6QSstwlqYQsd0kqIctdkkrIcpekErLcJamELHdJKiHLXZJKyHKXpBKy3CWphCx3SSohy12S\nSshyl6QSstwlqYQsd0kqIctdkkrIcpekErLcJamELHdJKiHLXZJKyHKXpBKy3CWphCx3SSohy12S\nSqhQuUfE9RGxPiK6I+Lzx1gfEfHVyvrnIuKy+keVJBU1bLlHRBtwL3ADsBi4NSIWDxl2A7Cg8mcJ\ncF+dc0qSalDkyP1KoDszN2TmYWA5cPOQMTcD38pBTwJnR8S5dc4qSSqoSLnPAjZXPe6pLKt1jCRp\nhIzoCdWIWBIRqyJiVW9v70huWpJawvUXz2TRzMkN387YAmO2AOdXPZ5dWVbrGDJzGbAMoKurK2tK\nKkkl8NVbf2VEtlPkyP1pYEFEzIuIduAWYMWQMSuA2ypXzbwb2J2Z2+qcVZJU0LBH7pnZHxF3Ao8C\nbcADmbkuIu6orF8KrARuBLqBA8DtjYssSRpOkWkZMnMlgwVevWxp1c8JfLq+0SRJJ8tPqEpSCVnu\nklRClrsklZDlLkklZLlLUgnF4IUuTdhwRC+wqYandAB9DYrTCOZtLPM2lnkb61TyXpCZncMNalq5\n1yoiVmVmV7NzFGXexjJvY5m3sUYir9MyklRClrskldBoKvdlzQ5QI/M2lnkby7yN1fC8o2bOXZJU\n3Gg6cpckFdRy5V7gZtw3V27C/Wzlxh/XNCNnVZ4T5q0ad0VE9EfER0cy3zFyDLd/3xsRuyv799mI\n+LNm5KzKM+z+rWR+NiLWRcRPRjrjkCzD7d/PVe3btRFxNCLOadGsZ0XEwxGxurJvm/ptrwXyTo2I\nf6j0w1MRcUkzclbleSAidkTE2uOsj4j4auW/57mIuKyuATKzZf4w+JXCLwPzgXZgNbB4yJhJ/Hw6\n6V3Ai62ct2rcYwx+s+ZHWzkv8F7gB83+Xagh79nA88CcyuPprZx3yPibgMdaNSvw34EvV37uBHYC\n7S2c9yvAFys/LwL+uVm/C5UMvwZcBqw9zvobgR8BAbwb+Pd6br/VjtyHvRl3Zu7Lyp4BJgLNPGlQ\n5ObhAJ8BvgvsGMlwx1A0b6sokvfjwEOZ+SpAZjZzH9e6f28FvjMiyX5RkawJTI6IYPCgaifQP7Ix\n31Ik72IGD6LIzBeBuRExY2Rj/lxmPs7gPjuem4Fv5aAngbMj4tx6bb/Vyr3QjbYj4sMR8SLwQ+AT\nI5TtWIbNGxGzgA8D941gruMpeiPzqytvE38UERePTLRjKpJ3ITA1In4cEc9ExG0jlu4XFb5RfERM\nAK5n8B/9ZiiS9evAO4GtwBrgjzJzYGTi/YIieVcDvwUQEVcCFzB4y89WVfj35WS0WrkXkpn/kJmL\ngN8E7m52nmH8FXBXE/9S1OqnDE5xvAv4GvC9JucZzljgcuBDwHXAFyJiYXMjFXIT8G+ZeaIju2a7\nDngWOA/4ZeDrETGluZFO6B4Gj36fZfDd8n8AR5sbqXkK3YlpBBW60fabMvPxiJgfER2Z2YzvlSiS\ntwtYPvjOlg7gxojoz8xmlOaweTNzT9XPKyPiGy2+f3uA1zNzP7A/Ih4HLgVeGpmIb1PL7+8tNG9K\nBoplvR24pzIN2h0RGxmcy35qZCK+TdHf3dth8GQlsBHYMFIBT0JNfVezZp5wOMYJhrEM/s+Yx89P\nmlw8ZMw7+PkJ1csqOyNaNe+Q8Q/S3BOqRfbvzKr9eyXwaivvXwanDf65MnYCsBa4pFXzVsadxeBc\n7MQW/124D/jzys8zKn/XOlo479lUTvgCf8DgfHZT9m9Vprkc/4Tqh3j7CdWn6rntljpyz2I34/4I\ncFtEHAEOAr+dlT3VonlbRsG8HwX+c0T0M7h/b2nl/ZuZL0TEI8BzwABwf2Ye89KzVshbGfph4B9z\n8N1GUxTMejfwYESsYbCA7srmvIMrmvedwN9ERALrgE82I+ubIuI7DF591hERPcAXgXHwVt6VDF4x\n0w0coPKuo27bb9LfW0lSA43KE6qSpBOz3CWphCx3SSohy12SSshyl6QSstwlqYQsd0kqIctdkkro\n/wM5zZOR9hkNfQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(precisions, recalls)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/07-ROC-Curve.ipynb b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/07-ROC-Curve.ipynb new file mode 100644 index 0000000..2c220f7 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/07-ROC-Curve.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ROC 曲线" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "\n", + "digits = datasets.load_digits()\n", + "X = digits.data\n", + "y = digits.target.copy()\n", + "\n", + "y[digits.target==9] = 1\n", + "y[digits.target!=9] = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "log_reg = LogisticRegression()\n", + "log_reg.fit(X_train, y_train)\n", + "decision_scores = log_reg.decision_function(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from playML.metrics import FPR, TPR\n", + "\n", + "fprs = []\n", + "tprs = []\n", + "thresholds = np.arange(np.min(decision_scores), np.max(decision_scores), 0.1)\n", + "for threshold in thresholds:\n", + " y_predict = np.array(decision_scores >= threshold, dtype='int')\n", + " fprs.append(FPR(y_test, y_predict))\n", + " tprs.append(TPR(y_test, y_predict))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAECZJREFUeJzt3X2QXXV9x/H3NxvCgzwEyEIgDyZIANMGVFZk1GqUqU34\nw8CUaYOMjlYnpYLaB2fgj47tjNOOrW1HLeBOhonWaWucKtWMXUydOuKMSElQBCMPswTMA2hCSEl5\nMmzy7R97rZeF5J7dPXfP3t++XzM7c885v9z7+c1JPjl77rnnRmYiSSrLrKYDSJLqZ7lLUoEsd0kq\nkOUuSQWy3CWpQJa7JBXIcpekAlnuklQgy12SCjS7qReeN29eLlmypKmXl6SedM899zyZmf2dxjVW\n7kuWLGHr1q1Nvbwk9aSI+FmVcZ6WkaQCWe6SVCDLXZIKZLlLUoEsd0kqUMdyj4gNEbEnIn5yhO0R\nEZ+LiOGIuC8i3lB/TEnSeFQ5cv8isOoo21cDy1o/64DPTz6WJGkyOl7nnpnfi4glRxmyBvhSjn5f\n310RMTcizsrMJ2rKqBnkzkee5K5H9jUdQ+qqgSWn8bbzOn4OaVLq+BDTAmBn2/Ku1rqXlXtErGP0\n6J7FixfX8NIqyQsvHuIj//oj9j17kIim00jdc+3bX9MT5V5ZZq4H1gMMDAz4zdx6iU0/fpx9zx7k\nXz70Jt5y7rym40g9rY6rZXYDi9qWF7bWSZVlJl/4/mOcf+ZJvPk1pzcdR+p5dZT7JuB9ratmLgWe\n9ny72o0cOtzx5wfb9/HAEwd4/1uWEJ6TkSat42mZiPgysBKYFxG7gL8AjgHIzEFgCLgcGAaeAz7Q\nrbDqPTd89T6+snVn54HA3BOO4YrXLehyImlmqHK1zNUdtidwXW2JVIwd+57j3+7ZyTsvOIPXL5rb\ncfzAktM4fk7fFCSTytfYLX9Vvg3ff5S+WcFfX7mC+acc13QcaUbx9gPqiv3PHuQrW3by7osWWOxS\nAzxy75Knn3+Rf7rzMX45cqjpKI146Of/y/MvHmLd285pOoo0I1nuXXLHw3v5h28/TN+sYKZe+/Hu\ni87m/PknNR1DmpEs9y4ZfZ8Zvv0nb+Oc/hMbTiNppvGcuyQVyCP3Cep0Lv3FQ95dQVJzLPcJ+Ntv\nPcgt332k0tjZs/zlSNLUs9zHKTO57Ye7WbHgFFb95vyjjj3tVXNYdNrxU5RMkn7Nch+nbY8f4OcH\nXuBP33UevzewqPMfkKQGeM5gnP7rgT1EwDsvOKPpKJJ0RB65V/CjHfv51rafAzB0/xO8btFc5p14\nbMOpJOnILPcKBu94hM3bfsGxs2cxK4IPrzy36UiSdFSWewWHE1571snc/rHfajqKJFXiOXdJKpBH\n7m2eP3iI5OUfPjp02A8kSeotlnvL4B2P8KnbHzzi9hULTpnCNJI0OZZ7y86nnuOEOX187LJlr7h9\nYMmpU5xIkibOcm9zwpw+/vDtr2k6hiRNmm+oSlKBij5y33j3Dob3PFNp7D0/29/lNJI0dYou9z//\n+k+IgDl91X5BufSc07ucSJKmRtHlfjiT61aey5+96/ymo0jSlPKcuyQVyHKXpAJZ7pJUIMtdkgpk\nuUtSgSx3SSpQEZdCDt3/BHc/+tTL1nszR0kzVRHl/unND7Fr/3Mcf0zfS9afesIxLD/r5IZSSVJz\nKpV7RKwCPgv0Abdm5qfGbD8F+Gdgces5/y4zv1Bz1iPKTC5fcRafXfv6qXpJSZrWOp5zj4g+4GZg\nNbAcuDoilo8Zdh3w08y8CFgJ/H1EzKk5qySpoipvqF4CDGfm9sw8CGwE1owZk8BJERHAicBTwEit\nSY/gmV+OcCg9uS5J7aqU+wJgZ9vyrta6djcBrwUeB+4HPpaZh2tJeBR3P/oUK/5yMzufep7Zs7zw\nR5J+pa5G/B3gXuBs4HXATRHxsncyI2JdRGyNiK179+6d9Iv+4sALZML17ziXj1527qSfT5JKUaXc\ndwOL2pYXtta1+wBwW44aBh4FLhj7RJm5PjMHMnOgv79/oplf5orXn82rT39Vbc8nSb2uSrlvAZZF\nxNLWm6RrgU1jxuwALgOIiDOB84HtdQaVJFXX8VLIzByJiOuBzYxeCrkhM7dFxLWt7YPAJ4EvRsT9\nQAA3ZOaTXcwtSTqKSte5Z+YQMDRm3WDb48eBd9UbTZI0UV5iIkkFstwlqUCWuyQVyHKXpAJZ7pJU\nIMtdkgpkuUtSgSx3SSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kqUKUv65iO\nvnHvbjbevbPpGJI0LfVsud/0nWF2PPUcF8w/iTNPPq7pOJI0rfRsuQNc9tozuOWai5uOIUnTjufc\nJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kqkOUuSQWy3CWpQJa7JBXIcpekAlnuklSgSuUe\nEasi4qGIGI6IG48wZmVE3BsR2yLijnpjSpLGo+NdISOiD7gZ+G1gF7AlIjZl5k/bxswFbgFWZeaO\niDijW4ElSZ1VOXK/BBjOzO2ZeRDYCKwZM+Y9wG2ZuQMgM/fUG1OSNB5Vyn0B0P6VR7ta69qdB5wa\nEd+NiHsi4n11BZQkjV9dX9YxG7gYuAw4HvhBRNyVmQ+3D4qIdcA6gMWLF9f00pKksaocue8GFrUt\nL2yta7cL2JyZz2bmk8D3gIvGPlFmrs/Mgcwc6O/vn2hmSVIHVcp9C7AsIpZGxBxgLbBpzJhvAG+N\niNkRcQLwJuCBeqNKkqrqeFomM0ci4npgM9AHbMjMbRFxbWv7YGY+EBHfAu4DDgO3ZuZPuhlcknRk\nlc65Z+YQMDRm3eCY5U8Dn64vmiRpovyEqiQVyHKXpAJZ7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBbLc\nJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kqkOUuSQWy3CWpQJa7JBXIcpekAlnuklQgy12S\nCmS5S1KBeq7cf3HgBT765R/x+P8833QUSZq2eq7ctz62n00/fpwzTz6Oleed0XQcSZqWZjcdYKIG\n33sx5515UtMxJGla6rkjd0lSZ5a7JBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkFqlTuEbEq\nIh6KiOGIuPEo494YESMRcVV9ESVJ49Wx3COiD7gZWA0sB66OiOVHGPc3wH/WHVKSND5VjtwvAYYz\nc3tmHgQ2AmteYdxHgK8Be2rMJ0magCrlvgDY2ba8q7Xu/0XEAuBK4PP1RZMkTVRdb6h+BrghMw8f\nbVBErIuIrRGxde/evTW9tCRprCp3hdwNLGpbXtha124A2BgRAPOAyyNiJDO/3j4oM9cD6wEGBgZy\noqElSUdXpdy3AMsiYimjpb4WeE/7gMxc+qvHEfFF4Jtji12SNHU6lntmjkTE9cBmoA/YkJnbIuLa\n1vbBLmeUJI1TpS/ryMwhYGjMulcs9cx8/+RjSZImw0+oSlKBLHdJKpDlLkkFstwlqUCWuyQVyHKX\npAJZ7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kq\nkOUuSQWy3CWpQJa7JBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkFstwlqUCWuyQVyHKXpAJZ\n7pJUoErlHhGrIuKhiBiOiBtfYfs1EXFfRNwfEXdGxEX1R5UkVdWx3COiD7gZWA0sB66OiOVjhj0K\nvD0zVwCfBNbXHVSSVF2VI/dLgOHM3J6ZB4GNwJr2AZl5Z2buby3eBSysN6YkaTyqlPsCYGfb8q7W\nuiP5IHD7ZEJJkiZndp1PFhHvYLTc33qE7euAdQCLFy+u86UlSW2qHLnvBha1LS9srXuJiLgQuBVY\nk5n7XumJMnN9Zg5k5kB/f/9E8kqSKqhS7luAZRGxNCLmAGuBTe0DImIxcBvw3sx8uP6YkqTx6Hha\nJjNHIuJ6YDPQB2zIzG0RcW1r+yDwCeB04JaIABjJzIHuxZYkHU2lc+6ZOQQMjVk32Pb4Q8CH6o0m\nSZooP6EqSQWy3CWpQJa7JBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkFstwlqUCWuyQVyHKX\npAJZ7pJUIMtdkgpkuUtSgSx3SSqQ5S5JBbLcJalAlrskFchyl6QCWe6SVCDLXZIKZLlLUoEsd0kq\nkOUuSQWy3CWpQJa7JBXIcpekAlnuklQgy12SCmS5S1KBeq7c559yHJevmM+Jx85uOookTVuVyj0i\nVkXEQxExHBE3vsL2iIjPtbbfFxFvqD/qqItffSq3XHMxZ889vlsvIUk9r2O5R0QfcDOwGlgOXB0R\ny8cMWw0sa/2sAz5fc05J0jhUOXK/BBjOzO2ZeRDYCKwZM2YN8KUcdRcwNyLOqjmrJKmiKuW+ANjZ\ntryrtW68Y4iIdRGxNSK27t27d7xZJUkVTekbqpm5PjMHMnOgv79/Kl9akmaUKuW+G1jUtrywtW68\nYyRJU6RKuW8BlkXE0oiYA6wFNo0Zswl4X+uqmUuBpzPziZqzSpIq6nixeGaORMT1wGagD9iQmdsi\n4trW9kFgCLgcGAaeAz7QvciSpE4qfRIoM4cYLfD2dYNtjxO4rt5okqSJitFebuCFI/YCP5vgH58H\nPFljnF7gnGcG5zwzTGbOr87MjlekNFbukxERWzNzoOkcU8k5zwzOeWaYijn33L1lJEmdWe6SVKBe\nLff1TQdogHOeGZzzzND1OffkOXdJ0tH16pG7JOkopnW5T6f7yE+VCnO+ICJ+EBG/jIiPN5GxbhXm\nfE1r/94fEXdGxEVN5KxThTmvac353tbN9t7aRM46dZpz27g3RsRIRFw1lfnqVmEfr4yIp1v7+N6I\n+EStATJzWv4w+mnYR4BzgDnAj4HlY8ZcDtwOBHAp8N9N556COZ8BvBH4K+DjTWeeojm/GTi19Xj1\nDNnPJ/Lr06YXAg82nbvbc24b9x1GPzR5VdO5u7yPVwLf7FaG6XzkPhPvI99xzpm5JzO3AC82EbAL\nqsz5zszc31q8i9Eb0/WyKnN+JlsNALwK6PU3x6r8ewb4CPA1YM9UhuuCqvPtmulc7rXdR76HlDaf\nKsY75w8y+ttaL6v6/QdXRsSDwH8AfzBF2bql45wjYgFwJWV8k1vVv9dvbp1+uz0ifqPOANO53KWX\niIh3MFruNzSdZSpk5r9n5gXAFcAnm84zBT4D3JCZh5sOMkV+CCzOzAuBfwS+XueTT+dyn4n3kS9t\nPlVUmnNEXAjcCqzJzH1TlK1bxrWfM/N7wDkRMa/bwbqoypwHgI0R8RhwFXBLRFwxNfFq13G+mXkg\nM59pPR4CjqlzH0/ncp+J95GvMufSdJxzRCwGbgPem5kPN5CxblXmfG5EROvxG4BjgV7+T63jnDNz\naWYuycwlwFeBD2dmrUezU6jKPp7fto8vYbSPa9vHlW7524ScgfeRrzLniJgPbAVOBg5HxB8z+i78\ngcaCT0LF/fwJ4HRGj+QARrKHbzRVcc6/y+iBy4vA88Dvt73B2nMqzrkYFed7FfBHETHC6D5eW+c+\n9hOqklSg6XxaRpI0QZa7JBXIcpekAlnuklQgy12SCmS5S1KBLHdJKpDlLkkF+j9jI3cd3SpnBQAA\nAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(fprs, tprs)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### scikit-learn中的ROC" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.metrics import roc_curve\n", + "\n", + "fprs, tprs, thresholds = roc_curve(y_test, decision_scores)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADotJREFUeJzt3W+IXXedx/H3ZxMLitaqGaUmzSa7xD8ja4uOjbuU3bqy\na1KQIPigrVgsSrZsKz5s2Qf2gU8Ud0HFagglW2SpWViLzS7RurBoF2qzTZfYNi2V2RTTxEKnrSjo\ngzL0uw/m1r07nck9M7lz79zffb9g4J5zfrnn+2PCJ7/8zu+ck6pCktSWPxh3AZKk4TPcJalBhrsk\nNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ3aOq4Tb9u2rXbt2jWu00vSRHr00UdfqKqZQe3G\nFu67du3i5MmT4zq9JE2kJL/o0s5pGUlqkOEuSQ0y3CWpQYa7JDXIcJekBg0M9yRHkjyf5IlVjifJ\nN5LMJ3ksyQeGX6YkaS26jNzvAfZd4Ph+YE/v5yDw7YsvS5J0MQauc6+qB5PsukCTA8B3aul9fQ8n\nuSzJ5VX13JBq1Drde+Is9586P+4yJC0z+85LufPj79vQcwxjzn078Gzf9rnevtdIcjDJySQnFxYW\nhnBqXcj9p87z5HO/GXcZksZgpHeoVtVh4DDA3Nycb+YegdnLL+Wf/+ZPx12GpBEbxsj9PHBF3/aO\n3j5J0pgMI9yPATf1Vs18GPi18+2SNF4Dp2WSfBe4FtiW5BxwJ/A6gKo6BBwHrgPmgd8BN29UsVrb\nRdInn/sNs5dfusEVSdqMuqyWuWHA8QJuHVpFuqBXL5J2Ce3Zyy/lwFUrXtuW1LixPfJX6+dFUkmD\n+PgBSWqQI/c+k3DTj/Pokrpw5N5nEm76cR5dUheO3JdxPltSCxy5S1KDDHdJatDUTsusdPHUi5WS\nWjG1I/eVLp56sVJSK6Z25A5ePJXUrqkduUtSy6Zq5N4/z+78uqSWTdXIvX+e3fl1SS2bqpE7OM8u\naTpM1chdkqaF4S5JDWp+WsaLqJKmUfMjdy+iSppGzY/cwYuokqZP8yN3SZpGzYzcV3uLkvPskqZR\nMyP31d6i5Dy7pGnUzMgdnFuXpFc1M3KXJP0fw12SGmS4S1KDDHdJapDhLkkNMtwlqUETvRTSh4JJ\n0someuTuQ8EkaWWdRu5J9gFfB7YAd1fVl5cdfzPwT8DO3nf+fVX945BrXZE3LknSaw0cuSfZAtwF\n7AdmgRuSzC5rdivwZFVdCVwL/EOSS4ZcqySpoy7TMlcD81V1pqpeBo4CB5a1KeBNSQK8EXgJWBxq\npZKkzrqE+3bg2b7tc719/b4JvBf4JfA48IWqemUoFa7i3hNnOfHMSxt5CkmaWMO6oPox4BTwTuAq\n4JtJXrN0JcnBJCeTnFxYWLioE766SsaLqJL0Wl3C/TxwRd/2jt6+fjcD99WSeeAZ4D3Lv6iqDlfV\nXFXNzczMrLfm39u7+63cuHfnRX+PJLWmS7g/AuxJsrt3kfR64NiyNmeBjwIkeQfwbuDMMAuVJHU3\ncClkVS0muQ14gKWlkEeq6nSSW3rHDwFfAu5J8jgQ4PaqemED65YkXUCnde5VdRw4vmzfob7PvwT+\nerilSZLWa6LvUJUkrcxwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnu\nktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUETGe73njjLiWdeGncZkrRpTWS4339q6f3cB67aPuZK\nJGlzmshwB9i7+63cuHfnuMuQpE1pYsNdkrQ6w12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCX\npAYZ7pLUIMNdkhpkuEtSgwx3SWpQp3BPsi/J00nmk9yxSptrk5xKcjrJT4ZbpiRpLbYOapBkC3AX\n8FfAOeCRJMeq6sm+NpcB3wL2VdXZJG/fqIIlSYN1GblfDcxX1Zmqehk4ChxY1uZG4L6qOgtQVc8P\nt0xJ0lp0CfftwLN92+d6+/q9C3hLkh8neTTJTcMqUJK0dgOnZdbwPR8EPgq8Hvhpkoer6uf9jZIc\nBA4C7NzpizYkaaN0GbmfB67o297R29fvHPBAVf22ql4AHgSuXP5FVXW4quaqam5mZma9NUuSBugS\n7o8Ae5LsTnIJcD1wbFmb+4FrkmxN8gZgL/DUcEuVJHU1cFqmqhaT3AY8AGwBjlTV6SS39I4fqqqn\nkvwQeAx4Bbi7qp7YyMIlSavrNOdeVceB48v2HVq2/VXgq8MrTZK0Xt6hKkkNMtwlqUGGuyQ1yHCX\npAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq\nkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgiQv3e0+c5cQzL427DEna1CYu3O8/dR6AA1dtH3MlkrR5\nTVy4A+zd/VZu3Ltz3GVI0qY1keEuSboww12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAZ1\nCvck+5I8nWQ+yR0XaPehJItJPjm8EiVJazUw3JNsAe4C9gOzwA1JZldp9xXgR8MuUpK0Nl1G7lcD\n81V1pqpeBo4CB1Zo93nge8DzQ6xPkrQOXcJ9O/Bs3/a53r7fS7Id+ATw7eGVJklar2FdUP0acHtV\nvXKhRkkOJjmZ5OTCwsKQTi1JWm5rhzbngSv6tnf09vWbA44mAdgGXJdksaq+39+oqg4DhwHm5uZq\nvUVLki6sS7g/AuxJspulUL8euLG/QVXtfvVzknuAf1se7JKk0RkY7lW1mOQ24AFgC3Ckqk4nuaV3\n/NAG1yhJWqMuI3eq6jhwfNm+FUO9qj5z8WVJki6Gd6hKUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJek\nBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ\n4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnu\nktSgTuGeZF+Sp5PMJ7ljheOfSvJYkseTPJTkyuGXKknqamC4J9kC3AXsB2aBG5LMLmv2DPAXVfUn\nwJeAw8MuVJLUXZeR+9XAfFWdqaqXgaPAgf4GVfVQVf2qt/kwsGO4ZUqS1qJLuG8Hnu3bPtfbt5rP\nAj+4mKIkSRdn6zC/LMlHWAr3a1Y5fhA4CLBz585hnlqS1KfLyP08cEXf9o7evv8nyfuBu4EDVfXi\nSl9UVYeraq6q5mZmZtZTrySpgy7h/giwJ8nuJJcA1wPH+hsk2QncB3y6qn4+/DIlSWsxcFqmqhaT\n3AY8AGwBjlTV6SS39I4fAr4IvA34VhKAxaqa27iyJUkX0mnOvaqOA8eX7TvU9/lzwOeGW5okab28\nQ1WSGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLc\nJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12S\nGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAZtHXcBazX7zkvHXYIkbXqdwj3JPuDrwBbg7qr6\n8rLj6R2/Dvgd8Jmq+u8h1wrAnR9/30Z8rSQ1ZeC0TJItwF3AfmAWuCHJ7LJm+4E9vZ+DwLeHXKck\naQ26zLlfDcxX1Zmqehk4ChxY1uYA8J1a8jBwWZLLh1yrJKmjLuG+HXi2b/tcb99a25DkYJKTSU4u\nLCystVZJUkcjXS1TVYeraq6q5mZmZkZ5akmaKl3C/TxwRd/2jt6+tbaRJI1Il3B/BNiTZHeSS4Dr\ngWPL2hwDbsqSDwO/rqrnhlyrJKmjgUshq2oxyW3AAywthTxSVaeT3NI7fgg4ztIyyHmWlkLevHEl\nS5IG6bTOvaqOsxTg/fsO9X0u4NbhliZJWq8s5fIYTpwsAL9Y5x/fBrwwxHImgX2eDvZ5OlxMn/+w\nqgauSBlbuF+MJCeram7cdYySfZ4O9nk6jKLPPjhMkhpkuEtSgyY13A+Pu4AxsM/TwT5Phw3v80TO\nuUuSLmxSR+6SpAvY1OGeZF+Sp5PMJ7ljheNJ8o3e8ceSfGAcdQ5Thz5/qtfXx5M8lOTKcdQ5TIP6\n3NfuQ0kWk3xylPVthC59TnJtklNJTif5yahrHLYOf7ffnORfk/ys1+eJvhkyyZEkzyd5YpXjG5tf\nVbUpf1i6G/Z/gD8CLgF+Bswua3Md8AMgwIeBE+OuewR9/jPgLb3P+6ehz33t/oOlm+k+Oe66R/B7\nvgx4EtjZ2377uOseQZ//DvhK7/MM8BJwybhrv4g+/znwAeCJVY5vaH5t5pH7ND5HfmCfq+qhqvpV\nb/Nhlh7SNsm6/J4BPg98D3h+lMVtkC59vhG4r6rOAlTVpPe7S58LeFPvzW5vZCncF0db5vBU1YMs\n9WE1G5pfmznch/Yc+Qmy1v58lqV/+SfZwD4n2Q58gnbe8NXl9/wu4C1Jfpzk0SQ3jay6jdGlz98E\n3gv8Engc+EJVvTKa8sZiQ/Nr4l6QrSVJPsJSuF8z7lpG4GvA7VX1ytKgbipsBT4IfBR4PfDTJA9X\n1c/HW9aG+hhwCvhL4I+Bf0/yn1X1m/GWNZk2c7hP43PkO/UnyfuBu4H9VfXiiGrbKF36PAcc7QX7\nNuC6JItV9f3RlDh0Xfp8Dnixqn4L/DbJg8CVwKSGe5c+3wx8uZYmpOeTPAO8B/iv0ZQ4chuaX5t5\nWmYanyM/sM9JdgL3AZ9uZBQ3sM9VtbuqdlXVLuBfgL+d4GCHbn+37weuSbI1yRuAvcBTI65zmLr0\n+SxL/1MhyTuAdwNnRlrlaG1ofm3akXtN4XPkO/b5i8DbgG/1RrKLNcEPXerY56Z06XNVPZXkh8Bj\nwCvA3VW14pK6SdDx9/wl4J4kj7O0guT2qprYp0Um+S5wLbAtyTngTuB1MJr88g5VSWrQZp6WkSSt\nk+EuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KD/hektxxcMN0dlwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(fprs, tprs)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ROC AUC" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.98304526748971188" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.metrics import roc_auc_score\n", + "\n", + "roc_auc_score(y_test, decision_scores)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LinearRegression.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LinearRegression.py new file mode 100644 index 0000000..c42cf64 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LinearRegression.py @@ -0,0 +1,116 @@ +import numpy as np +from .metrics import r2_score + +class LinearRegression: + + def __init__(self): + """初始化Linear Regression模型""" + self.coef_ = None + self.intercept_ = None + self._theta = None + + def fit_normal(self, X_train, y_train): + """根据训练数据集X_train, y_train训练Linear Regression模型""" + assert X_train.shape[0] == y_train.shape[0], \ + "the size of X_train must be equal to the size of y_train" + + X_b = np.hstack([np.ones((len(X_train), 1)), X_train]) + self._theta = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y_train) + + self.intercept_ = self._theta[0] + self.coef_ = self._theta[1:] + + return self + + def fit_bgd(self, X_train, y_train, eta=0.01, n_iters=1e4): + """根据训练数据集X_train, y_train, 使用梯度下降法训练Linear Regression模型""" + assert X_train.shape[0] == y_train.shape[0], \ + "the size of X_train must be equal to the size of y_train" + + def J(theta, X_b, y): + try: + return np.sum((y - X_b.dot(theta)) ** 2) / len(y) + except: + return float('inf') + + def dJ(theta, X_b, y): + return X_b.T.dot(X_b.dot(theta) - y) * 2. / len(y) + + def gradient_descent(X_b, y, initial_theta, eta, n_iters=1e4, epsilon=1e-8): + + theta = initial_theta + cur_iter = 0 + + while cur_iter < n_iters: + gradient = dJ(theta, X_b, y) + last_theta = theta + theta = theta - eta * gradient + if (abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon): + break + + cur_iter += 1 + + return theta + + X_b = np.hstack([np.ones((len(X_train), 1)), X_train]) + initial_theta = np.zeros(X_b.shape[1]) + self._theta = gradient_descent(X_b, y_train, initial_theta, eta, n_iters) + + self.intercept_ = self._theta[0] + self.coef_ = self._theta[1:] + + return self + + def fit_sgd(self, X_train, y_train, n_iters=50, t0=5, t1=50): + """根据训练数据集X_train, y_train, 使用梯度下降法训练Linear Regression模型""" + assert X_train.shape[0] == y_train.shape[0], \ + "the size of X_train must be equal to the size of y_train" + assert n_iters >= 1 + + def dJ_sgd(theta, X_b_i, y_i): + return X_b_i * (X_b_i.dot(theta) - y_i) * 2. + + def sgd(X_b, y, initial_theta, n_iters=5, t0=5, t1=50): + + def learning_rate(t): + return t0 / (t + t1) + + theta = initial_theta + m = len(X_b) + for i_iter in range(n_iters): + indexes = np.random.permutation(m) + X_b_new = X_b[indexes,:] + y_new = y[indexes] + for i in range(m): + gradient = dJ_sgd(theta, X_b_new[i], y_new[i]) + theta = theta - learning_rate(i_iter * m + i) * gradient + + return theta + + X_b = np.hstack([np.ones((len(X_train), 1)), X_train]) + initial_theta = np.random.randn(X_b.shape[1]) + self._theta = sgd(X_b, y_train, initial_theta, n_iters, t0, t1) + + self.intercept_ = self._theta[0] + self.coef_ = self._theta[1:] + + return self + + def predict(self, X_predict): + """给定待预测数据集X_predict，返回表示X_predict的结果向量""" + assert self.intercept_ is not None and self.coef_ is not None, \ + "must fit before predict!" + assert X_predict.shape[1] == len(self.coef_), \ + "the feature number of X_predict must be equal to X_train" + + X_b = np.hstack([np.ones((len(X_predict), 1)), X_predict]) + return X_b.dot(self._theta) + + def score(self, X_test, y_test): + """根据测试数据集 X_test 和 y_test 确定当前模型的准确度""" + + y_predict = self.predict(X_test) + return r2_score(y_test, y_predict) + + def __repr__(self): + return "LinearRegression()" diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LogisticRegression.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LogisticRegression.py new file mode 100644 index 0000000..4204ec5 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/LogisticRegression.py @@ -0,0 +1,82 @@ +import numpy as np +from .metrics import accuracy_score + +class LogisticRegression: + + def __init__(self): + """初始化Logistic Regression模型""" + self.coef_ = None + self.intercept_ = None + self._theta = None + + def _sigmoid(self, t): + return 1. / (1. + np.exp(-t)) + + def fit(self, X_train, y_train, eta=0.01, n_iters=1e4): + """根据训练数据集X_train, y_train, 使用梯度下降法训练Logistic Regression模型""" + assert X_train.shape[0] == y_train.shape[0], \ + "the size of X_train must be equal to the size of y_train" + + def J(theta, X_b, y): + y_hat = self._sigmoid(X_b.dot(theta)) + try: + return - np.sum(y*np.log(y_hat) + (1-y)*np.log(1-y_hat)) / len(y) + except: + return float('inf') + + def dJ(theta, X_b, y): + return X_b.T.dot(self._sigmoid(X_b.dot(theta)) - y) / len(y) + + def gradient_descent(X_b, y, initial_theta, eta, n_iters=1e4, epsilon=1e-8): + + theta = initial_theta + cur_iter = 0 + + while cur_iter < n_iters: + gradient = dJ(theta, X_b, y) + last_theta = theta + theta = theta - eta * gradient + if (abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon): + break + + cur_iter += 1 + + return theta + + X_b = np.hstack([np.ones((len(X_train), 1)), X_train]) + initial_theta = np.zeros(X_b.shape[1]) + self._theta = gradient_descent(X_b, y_train, initial_theta, eta, n_iters) + + self.intercept_ = self._theta[0] + self.coef_ = self._theta[1:] + + return self + + def predict_proba(self, X_predict): + """给定待预测数据集X_predict，返回表示X_predict的结果概率向量""" + assert self.intercept_ is not None and self.coef_ is not None, \ + "must fit before predict!" + assert X_predict.shape[1] == len(self.coef_), \ + "the feature number of X_predict must be equal to X_train" + + X_b = np.hstack([np.ones((len(X_predict), 1)), X_predict]) + return self._sigmoid(X_b.dot(self._theta)) + + def predict(self, X_predict): + """给定待预测数据集X_predict，返回表示X_predict的结果向量""" + assert self.intercept_ is not None and self.coef_ is not None, \ + "must fit before predict!" + assert X_predict.shape[1] == len(self.coef_), \ + "the feature number of X_predict must be equal to X_train" + + proba = self.predict_proba(X_predict) + return np.array(proba >= 0.5, dtype='int') + + def score(self, X_test, y_test): + """根据测试数据集 X_test 和 y_test 确定当前模型的准确度""" + + y_predict = self.predict(X_test) + return accuracy_score(y_test, y_predict) + + def __repr__(self): + return "LogisticRegression()" diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/PCA.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/PCA.py new file mode 100644 index 0000000..b812127 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/PCA.py @@ -0,0 +1,71 @@ +import numpy as np + + +class PCA: + + def __init__(self, n_components): + """初始化PCA""" + assert n_components >= 1, "n_components must be valid" + self.n_components = n_components + self.components_ = None + + def fit(self, X, eta=0.01, n_iters=1e4): + """获得数据集X的前n个主成分""" + assert self.n_components <= X.shape[1], \ + "n_components must not be greater than the feature number of X" + + def demean(X): + return X - np.mean(X, axis=0) + + def f(w, X): + return np.sum((X.dot(w) ** 2)) / len(X) + + def df(w, X): + return X.T.dot(X.dot(w)) * 2. / len(X) + + def direction(w): + return w / np.linalg.norm(w) + + def first_component(X, initial_w, eta=0.01, n_iters=1e4, epsilon=1e-8): + + w = direction(initial_w) + cur_iter = 0 + + while cur_iter < n_iters: + gradient = df(w, X) + last_w = w + w = w + eta * gradient + w = direction(w) + if (abs(f(w, X) - f(last_w, X)) < epsilon): + break + + cur_iter += 1 + + return w + + X_pca = demean(X) + self.components_ = np.empty(shape=(self.n_components, X.shape[1])) + for i in range(self.n_components): + initial_w = np.random.random(X_pca.shape[1]) + w = first_component(X_pca, initial_w, eta, n_iters) + self.components_[i,:] = w + + X_pca = X_pca - X_pca.dot(w).reshape(-1, 1) * w + + return self + + def transform(self, X): + """将给定的X，映射到各个主成分分量中""" + assert X.shape[1] == self.components_.shape[1] + + return X.dot(self.components_.T) + + def inverse_transform(self, X): + """将给定的X，反向映射回原来的特征空间""" + assert X.shape[1] == self.components_.shape[0] + + return X.dot(self.components_) + + def __repr__(self): + return "PCA(n_components=%d)" % self.n_components + diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/SimpleLinearRegression.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/SimpleLinearRegression.py new file mode 100644 index 0000000..a133e4d --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/SimpleLinearRegression.py @@ -0,0 +1,47 @@ +import numpy as np +from .metrics import r2_score + + +class SimpleLinearRegression: + + def __init__(self): + """初始化Simple Linear Regression模型""" + self.a_ = None + self.b_ = None + + def fit(self, x_train, y_train): + """根据训练数据集x_train, y_train训练Simple Linear Regression模型""" + assert x_train.ndim == 1, \ + "Simple Linear Regressor can only solve single feature training data." + assert len(x_train) == len(y_train), \ + "the size of x_train must be equal to the size of y_train" + + x_mean = np.mean(x_train) + y_mean = np.mean(y_train) + + self.a_ = (x_train - x_mean).dot(y_train - y_mean) / (x_train - x_mean).dot(x_train - x_mean) + self.b_ = y_mean - self.a_ * x_mean + + return self + + def predict(self, x_predict): + """给定待预测数据集x_predict，返回表示x_predict的结果向量""" + assert x_predict.ndim == 1, \ + "Simple Linear Regressor can only solve single feature training data." + assert self.a_ is not None and self.b_ is not None, \ + "must fit before predict!" + + return np.array([self._predict(x) for x in x_predict]) + + def _predict(self, x_single): + """给定单个待预测数据x，返回x的预测结果值""" + return self.a_ * x_single + self.b_ + + def score(self, x_test, y_test): + """根据测试数据集 x_test 和 y_test 确定当前模型的准确度""" + + y_predict = self.predict(x_test) + return r2_score(y_test, y_predict) + + def __repr__(self): + return "SimpleLinearRegression()" diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/__init__.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/kNN.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/kNN.py new file mode 100644 index 0000000..f8dee73 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/kNN.py @@ -0,0 +1,60 @@ +import numpy as np +from math import sqrt +from collections import Counter +from .metrics import accuracy_score + + +class KNNClassifier: + + def __init__(self, k): + """初始化kNN分类器""" + assert k >= 1, "k must be valid" + self.k = k + self._X_train = None + self._y_train = None + + def fit(self, X_train, y_train): + """根据训练数据集X_train和y_train训练kNN分类器""" + assert X_train.shape[0] == y_train.shape[0], \ + "the size of X_train must be equal to the size of y_train" + assert self.k <= X_train.shape[0], \ + "the size of X_train must be at least k." + + self._X_train = X_train + self._y_train = y_train + return self + + def predict(self, X_predict): + """给定待预测数据集X_predict，返回表示X_predict的结果向量""" + assert self._X_train is not None and self._y_train is not None, \ + "must fit before predict!" + assert X_predict.shape[1] == self._X_train.shape[1], \ + "the feature number of X_predict must be equal to X_train" + + y_predict = [self._predict(x) for x in X_predict] + return np.array(y_predict) + + def _predict(self, x): + """给定单个待预测数据x，返回x的预测结果值""" + assert x.shape[0] == self._X_train.shape[1], \ + "the feature number of x must be equal to X_train" + + distances = [sqrt(np.sum((x_train - x) ** 2)) + for x_train in self._X_train] + nearest = np.argsort(distances) + + topK_y = [self._y_train[i] for i in nearest[:self.k]] + votes = Counter(topK_y) + + return votes.most_common(1)[0][0] + + def score(self, X_test, y_test): + """根据测试数据集 X_test 和 y_test 确定当前模型的准确度""" + + y_predict = self.predict(X_test) + return accuracy_score(y_test, y_predict) + + def __repr__(self): + return "KNN(k=%d)" % self.k + + diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/metrics.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/metrics.py new file mode 100644 index 0000000..5d15a04 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/metrics.py @@ -0,0 +1,114 @@ +import numpy as np +from math import sqrt + + +def accuracy_score(y_true, y_predict): + """计算y_true和y_predict之间的准确率""" + assert len(y_true) == len(y_predict), \ + "the size of y_true must be equal to the size of y_predict" + + return np.sum(y_true == y_predict) / len(y_true) + + +def mean_squared_error(y_true, y_predict): + """计算y_true和y_predict之间的MSE""" + assert len(y_true) == len(y_predict), \ + "the size of y_true must be equal to the size of y_predict" + + return np.sum((y_true - y_predict)**2) / len(y_true) + + +def root_mean_squared_error(y_true, y_predict): + """计算y_true和y_predict之间的RMSE""" + + return sqrt(mean_squared_error(y_true, y_predict)) + + +def mean_absolute_error(y_true, y_predict): + """计算y_true和y_predict之间的MAE""" + assert len(y_true) == len(y_predict), \ + "the size of y_true must be equal to the size of y_predict" + + return np.sum(np.absolute(y_true - y_predict)) / len(y_true) + + +def r2_score(y_true, y_predict): + """计算y_true和y_predict之间的R Square""" + + return 1 - mean_squared_error(y_true, y_predict)/np.var(y_true) + + +def TN(y_true, y_predict): + assert len(y_true) == len(y_predict) + return np.sum((y_true == 0) & (y_predict == 0)) + + +def FP(y_true, y_predict): + assert len(y_true) == len(y_predict) + return np.sum((y_true == 0) & (y_predict == 1)) + + +def FN(y_true, y_predict): + assert len(y_true) == len(y_predict) + return np.sum((y_true == 1) & (y_predict == 0)) + + +def TP(y_true, y_predict): + assert len(y_true) == len(y_predict) + return np.sum((y_true == 1) & (y_predict == 1)) + + +def confusion_matrix(y_true, y_predict): + return np.array([ + [TN(y_true, y_predict), FP(y_true, y_predict)], + [FN(y_true, y_predict), TP(y_true, y_predict)] + ]) + + +def precision_score(y_true, y_predict): + assert len(y_true) == len(y_predict) + tp = TP(y_true, y_predict) + fp = FP(y_true, y_predict) + try: + return tp / (tp + fp) + except: + return 0.0 + + +def recall_score(y_true, y_predict): + assert len(y_true) == len(y_predict) + tp = TP(y_true, y_predict) + fn = FN(y_true, y_predict) + try: + return tp / (tp + fn) + except: + return 0.0 + + +def f1_score(y_true, y_predict): + precision = precision_score(y_true, y_predict) + recall = recall_score(y_true, y_predict) + + try: + return 2. * precision * recall / (precision + recall) + except: + return 0. + + +def TPR(y_true, y_predict): + tp = TP(y_true, y_predict) + fn = FN(y_true, y_predict) + try: + return tp / (tp + fn) + except: + return 0. + + +def FPR(y_true, y_predict): + fp = FP(y_true, y_predict) + tn = TN(y_true, y_predict) + try: + return fp / (fp + tn) + except: + return 0. + \ No newline at end of file diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/model_selection.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/model_selection.py new file mode 100644 index 0000000..878e7ed --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/model_selection.py @@ -0,0 +1,26 @@ +import numpy as np + + +def train_test_split(X, y, test_ratio=0.2, seed=None): + """将数据 X 和 y 按照test_ratio分割成X_train, X_test, y_train, y_test""" + assert X.shape[0] == y.shape[0], \ + "the size of X must be equal to the size of y" + assert 0.0 <= test_ratio <= 1.0, \ + "test_ration must be valid" + + if seed: + np.random.seed(seed) + + shuffled_indexes = np.random.permutation(len(X)) + + test_size = int(len(X) * test_ratio) + test_indexes = shuffled_indexes[:test_size] + train_indexes = shuffled_indexes[test_size:] + + X_train = X[train_indexes] + y_train = y[train_indexes] + + X_test = X[test_indexes] + y_test = y[test_indexes] + + return X_train, X_test, y_train, y_test diff --git a/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/preprocessing.py b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/preprocessing.py new file mode 100644 index 0000000..e95f89a --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/07-ROC-Curve/playML/preprocessing.py @@ -0,0 +1,30 @@ +import numpy as np + + +class StandardScaler: + + def __init__(self): + self.mean_ = None + self.scale_ = None + + def fit(self, X): + """根据训练数据集X获得数据的均值和方差""" + assert X.ndim == 2, "The dimension of X must be 2" + + self.mean_ = np.array([np.mean(X[:,i]) for i in range(X.shape[1])]) + self.scale_ = np.array([np.std(X[:,i]) for i in range(X.shape[1])]) + + return self + + def transform(self, X): + """将X根据这个StandardScaler进行均值方差归一化处理""" + assert X.ndim == 2, "The dimension of X must be 2" + assert self.mean_ is not None and self.scale_ is not None, \ + "must fit before transform!" + assert X.shape[1] == len(self.mean_), \ + "the feature number of X must be equal to mean_ and std_" + + resX = np.empty(shape=X.shape, dtype=float) + for col in range(X.shape[1]): + resX[:,col] = (X[:,col] - self.mean_[col]) / self.scale_[col] + return resX diff --git a/machinelearning/classificationPerformanceMeasures/08-Confusion-Matrix-in-Multiclass-Classification/08-Confusion-Matrix-in-Multiclass-Classification.ipynb b/machinelearning/classificationPerformanceMeasures/08-Confusion-Matrix-in-Multiclass-Classification/08-Confusion-Matrix-in-Multiclass-Classification.ipynb new file mode 100644 index 0000000..8bed089 --- /dev/null +++ b/machinelearning/classificationPerformanceMeasures/08-Confusion-Matrix-in-Multiclass-Classification/08-Confusion-Matrix-in-Multiclass-Classification.ipynb @@ -0,0 +1,230 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 多分类问题中的混淆矩阵" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "\n", + "digits = datasets.load_digits()\n", + "X = digits.data\n", + "y = digits.target" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.8, random_state=666)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.93115438108484005" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "log_reg = LogisticRegression()\n", + "log_reg.fit(X_train, y_train)\n", + "log_reg.score(X_test, y_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "y_predict = log_reg.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Target is multiclass but average='binary'. Please choose another average setting.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprecision_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mprecision_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_predict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/Users/yuanzhang/anaconda/lib/python3.6/site-packages/sklearn/metrics/classification.py\u001b[0m in \u001b[0;36mprecision_score\u001b[0;34m(y_true, y_pred, labels, pos_label, average, sample_weight)\u001b[0m\n\u001b[1;32m 1237\u001b[0m \u001b[0maverage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maverage\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1238\u001b[0m \u001b[0mwarn_for\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'precision'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m sample_weight=sample_weight)\n\u001b[0m\u001b[1;32m 1240\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1241\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/Users/yuanzhang/anaconda/lib/python3.6/site-packages/sklearn/metrics/classification.py\u001b[0m in \u001b[0;36mprecision_recall_fscore_support\u001b[0;34m(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight)\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m raise ValueError(\"Target is %s but average='binary'. Please \"\n\u001b[0;32m-> 1018\u001b[0;31m \"choose another average setting.\" % y_type)\n\u001b[0m\u001b[1;32m 1019\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mpos_label\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1020\u001b[0m warnings.warn(\"Note that pos_label (set to %r) is ignored when \"\n", + "\u001b[0;31mValueError\u001b[0m: Target is multiclass but average='binary'. Please choose another average setting." + ] + } + ], + "source": [ + "from sklearn.metrics import precision_score\n", + "\n", + "precision_score(y_test, y_predict)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.93115438108484005" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "precision_score(y_test, y_predict, average=\"micro\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([[147, 0, 1, 0, 0, 1, 0, 0, 0, 0],\n", + " [ 0, 123, 1, 2, 0, 0, 0, 3, 4, 10],\n", + " [ 0, 0, 134, 1, 0, 0, 0, 0, 1, 0],\n", + " [ 0, 0, 0, 138, 0, 5, 0, 1, 5, 0],\n", + " [ 2, 5, 0, 0, 139, 0, 0, 3, 0, 1],\n", + " [ 1, 3, 1, 0, 0, 146, 0, 0, 1, 0],\n", + " [ 0, 2, 0, 0, 0, 1, 131, 0, 2, 0],\n", + " [ 0, 0, 0, 1, 0, 0, 0, 132, 1, 2],\n", + " [ 1, 9, 2, 3, 2, 4, 0, 0, 115, 4],\n", + " [ 0, 1, 0, 5, 0, 3, 0, 2, 2, 134]])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.metrics import confusion_matrix\n", + "\n", + "confusion_matrix(y_test, y_predict)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACttJREFUeJzt3U2IXfUZx/HfLzOjY2JjhATETOhkUSwqlMhQokIWRqSt\nUjddWFCom9m0GkUQ7cZlNyK6KEKIdVOxi5hFkaIW1EUXhk5eQJOxINrmVZ0u4qgQMi9PF/cGX5LO\nPQP3uWdunu8HhMx48ufJ5H7n3Htzzn8cEQJQy7q2BwAweIQPFET4QEGEDxRE+EBBhA8U1Fr4tn9m\n+1+2P7L9VFtzNGV7m+13bB+3fcz2nrZnasL2iO0jtl9ve5YmbG+yvd/2h7Znbd/e9ky92H68+5j4\nwPartsfbnqmXVsK3PSLpj5J+LulmSb+2fXMbs6zCoqQnIuJmSTsl/XYIZpakPZJm2x5iFV6Q9EZE\n/FjST7TGZ7e9VdKjkqYi4lZJI5IeaHeq3to64/9U0kcR8XFEXJD0F0n3tzRLIxFxNiIOd3/9pToP\nyK3tTrUy2xOS7pW0r+1ZmrB9naRdkl6SpIi4EBHn2p2qkVFJ19gelbRe0pmW5+mprfC3Sjr5rY9P\naY1H9G22JyXtkHSw3Ul6el7Sk5KW2x6koe2S5iS93H15ss/2hraHWklEnJb0rKQTks5K+iIi3mp3\nqt54c2+VbF8r6TVJj0XEfNvz/D+275P0eUQcanuWVRiVdJukFyNih6SvJa3p939sX6/Os9Xtkm6U\ntMH2g+1O1Vtb4Z+WtO1bH090P7em2R5TJ/pXIuJA2/P0cKekX9r+tzovpe6y/ed2R+rplKRTEXHx\nmdR+db4RrGV3S/okIuYiYkHSAUl3tDxTT22F/09JP7K93fZV6rwZ8teWZmnEttV57TkbEc+1PU8v\nEfF0RExExKQ6X9+3I2JNn4ki4lNJJ23f1P3UbknHWxypiROSdtpe332M7NYaf0NS6jy1GriIWLT9\nO0lvqvMu6J8i4lgbs6zCnZIekvS+7aPdz/0+Iv7W4kxXokckvdI9IXws6eGW51lRRBy0vV/SYXX+\n5eeIpL3tTtWbuS0XqIc394CCCB8oiPCBgggfKIjwgYJaD9/2dNszrMawzSsx8yAM27ythy9pqL5g\nGr55JWYehKGady2ED2DAUi7g2bx5c0xOTjY6dm5uTlu2bGl07KFDw3S/CdCOiHCvY1Iu2Z2cnNTM\nzEzf1+1cCp0ja+1hvDKSr8WVj6f6QEGEDxRE+EBBhA8URPhAQY3CH7Y98AGsrGf4Q7oHPoAVNDnj\nD90e+ABW1iT8od4DH8Cl+vbmnu1p2zO2Z+bm5vq1LIAETcJvtAd+ROyNiKmImGp67T2AdjQJf+j2\nwAewsp436QzpHvgAVtDo7rzuD43gB0cAVwiu3AMKInygIMIHCiJ8oCDCBwpK2WzTdsrmaseP5/2o\n9FtuuSVl3XXr8r63Li0tpa09bMbGxlLWzdznccOGDX1fc35+XouLiz2H5owPFET4QEGEDxRE+EBB\nhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGE\nDxRE+EBBQ7W9dqbPPvssZd0bbrghZV1Jyvi7w3dlbq+d9fcXEWyvDeBShA8URPhAQYQPFET4QEGE\nDxRE+EBBPcO3vc32O7aP2z5me88gBgOQZ7TBMYuSnoiIw7Z/IOmQ7b9HRN4PqweQqucZPyLORsTh\n7q+/lDQraWv2YADyrOo1vu1JSTskHcwYBsBgNHmqL0myfa2k1yQ9FhHzl/n/05Km+zgbgCSNbtKx\nPSbpdUlvRsRzDY4furtHuEkHl1P2Jh13/uQvSZptEj2Ata/Ja/w7JT0k6S7bR7v//SJ5LgCJer7G\nj4h/SMp7vgNg4LhyDyiI8IGCCB8oiPCBgggfKIhddpOdO3cube1NmzalrDs+Pp6y7vnz51PWlfIu\ntLn66qtT1pXyvh7ssgvgsggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCB\ngggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHygobXvtkZGRvq87NjbW9zUvytz6Ocv8/HzK\nuhs3bkxZN1PWY2NhYSFlXSlnS/CIYHttAJdH+EBBhA8URPhAQYQPFET4QEGEDxTUOHzbI7aP2H49\ncyAA+VZzxt8jaTZrEACD0yh82xOS7pW0L3ccAIPQ9Iz/vKQnJS0nzgJgQHqGb/s+SZ9HxKEex03b\nnrE907fpAKToeZOO7T9IekjSoqRxSRslHYiIB1f4PdykMwDcpPMNbtLp6NtNOhHxdERMRMSkpAck\nvb1S9ADWPv4dHyhodDUHR8S7kt5NmQTAwHDGBwoifKAgwgcKInygIMIHCkrbZTfj4oTR0VX9I8Sq\nLC4upqyb8fXNtrycc2X2unXDd57JeBxflPXYYJddAJdF+EBBhA8URPhAQYQPFET4QEGEDxRE+EBB\nhA8URPhAQYQPFET4QEGEDxRE+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBhA8UlLZtbcYOolm7v0rD\nuRtulowfcS5JJ0+eTFlXkrZt25aybubOwEtLS2lr98IZHyiI8IGCCB8oiPCBgggfKIjwgYIIHyio\nUfi2N9neb/tD27O2b88eDECephfwvCDpjYj4le2rJK1PnAlAsp7h275O0i5Jv5GkiLgg6ULuWAAy\nNXmqv13SnKSXbR+xvc/2huS5ACRqEv6opNskvRgROyR9Lemp7x9ke9r2jO2ZPs8IoM+ahH9K0qmI\nONj9eL863wi+IyL2RsRUREz1c0AA/dcz/Ij4VNJJ2zd1P7Vb0vHUqQCkavqu/iOSXum+o/+xpIfz\nRgKQrVH4EXFUEk/hgSsEV+4BBRE+UBDhAwURPlAQ4QMFET5QkDO2lbbNXtVdttPWZkvwb5w+fTpl\n3YmJiZR1pZytu5eWlhQRPR90nPGBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGC\nCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYLSdtnN2F1248aNfV/zoq+++ipl3Yyd\nVC9aXl5OWXdkZCRl3QsXLqSsm+m9995LW3vXrl19X3NhYUHLy8vssgvgUoQPFET4QEGEDxRE+EBB\nhA8URPhAQY3Ct/247WO2P7D9qu3x7MEA5OkZvu2tkh6VNBURt0oakfRA9mAA8jR9qj8q6Rrbo5LW\nSzqTNxKAbD3Dj4jTkp6VdELSWUlfRMRb2YMByNPkqf71ku6XtF3SjZI22H7wMsdN256xPdP/MQH0\nU5On+ndL+iQi5iJiQdIBSXd8/6CI2BsRUxEx1e8hAfRXk/BPSNppe707t9ztljSbOxaATE1e4x+U\ntF/SYUnvd3/P3uS5ACQabXJQRDwj6ZnkWQAMCFfuAQURPlAQ4QMFET5QEOEDBRE+UFDa9tp9X7Sz\nbsaykqSMr4MkjY/n3cF8/vz5lHXHxsZS1l1YWEhZV8rbEjzTmTP9v9ftnnvu0dGjR9leG8ClCB8o\niPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI\n8IGCCB8oiPCBgrJ22Z2T9J+Gh2+W9N++D5Fn2OaVmHkQ1sq8P4yILb0OSgl/NWzPRMRUq0OswrDN\nKzHzIAzbvDzVBwoifKCgtRD+3rYHWKVhm1di5kEYqnlbf40PYPDWwhkfwIARPlAQ4QMFET5QEOED\nBf0PDFK2BTKTWFsAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "cfm = confusion_matrix(y_test, y_predict)\n", + "plt.matshow(cfm, cmap=plt.cm.gray)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACvZJREFUeJzt3U+IXfUZxvHnSWaCJhYVppqZRJosikHEkjDWPwEXxkXb\nBEXShQWFuskm1SiCaEQFwZ2IgkUYkrox6CLJokixFtRFN6HJRIjJKIiamPmDyaIq2WTGebuYG4gm\nnXNGzu+ee32/HxDMeHx5CfPNOffm3DOOCAHIZVnbCwDoPsIHEiJ8ICHCBxIifCAhwgcSai1827+z\n/antz2w/1dYeddm+wfYHtk/YPm57V9s71WF7ue2jtt9pe5c6bF9je7/tT2xP2L6j7Z2q2H688z3x\nse23bF/R9k5VWgnf9nJJf5X0e0k3SfqT7Zva2GUJ5iQ9ERE3Sbpd0s4+2FmSdkmaaHuJJXhV0rsR\nsUHSb9Tju9teI+lRSaMRcbOk5ZIeaHeram2d8X8r6bOI+Dwizkt6W9J9Le1SS0RMR8R459+/08I3\n5Jp2t1qc7bWStkra0/Yuddi+WtJdkvZKUkScj4j/trtVLQOSrrQ9IGmlpKmW96nUVvhrJH110a9P\nq8cjupjtdZI2SjrU7iaVXpH0pKT5thepab2kM5Le6Lw82WN7VdtLLSYiJiW9JOmUpGlJ30TEe+1u\nVY0395bI9lWSDkh6LCK+bXuf/8f2NklfR8SRtndZggFJmyS9HhEbJZ2T1NPv/9i+VgtXq+sljUha\nZfvBdreq1lb4k5JuuOjXaztf62m2B7UQ/b6IONj2PhU2S7rX9pdaeCl1t+03212p0mlJpyPiwpXU\nfi38QdDL7pH0RUSciYhZSQcl3dnyTpXaCv8/kn5te73tFVp4M+TvLe1Si21r4bXnRES83PY+VSLi\n6YhYGxHrtPD7+35E9PSZKCJmJH1l+8bOl7ZIOtHiSnWcknS77ZWd75Et6vE3JKWFS6uui4g523+R\n9E8tvAv6t4g43sYuS7BZ0kOSjtn+qPO13RHxjxZ3+jl6RNK+zgnhc0kPt7zPoiLikO39ksa18Dc/\nRyWNtbtVNfOxXCAf3twDEiJ8ICHCBxIifCAhwgcSaj182zva3mEp+m1fiZ27od/2bT18SX31G6b+\n21di527oq317IXwAXVbkBh7bfXdX0ODgYK3j5ufntWxZ/T8vZ2dnf+pKrRkYKHND59zcXJG5+KGI\ncNUxrdyy24uuu+66InMnJ3v+s0eXGBoaKjJ3ZmamyFwsHZf6QEKEDyRE+EBChA8kRPhAQrXC77dn\n4ANYXGX4ffoMfACLqHPG77tn4ANYXJ3w+/oZ+AAu1dide51PJ/XVBxWArOqEX+sZ+BExps7TRfvx\nXn0gkzqX+n33DHwAi6s84/fpM/ABLKLWa/zOD43gB0cAPxPcuQckRPhAQoQPJET4QEKEDyTEM/cK\n27RpU7HZ4+PjReb247Pxtm/fXmTu6tWri8yVpNdee63xmaOjo7WO44wPJET4QEKEDyRE+EBChA8k\nRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE\n+EBCjmj+R9nbbn5onxoeHi42e3p6uthsLBgZGSk2e2pqqsjciHDVMZzxgYQIH0iI8IGECB9IiPCB\nhAgfSIjwgYQqw7d9g+0PbJ+wfdz2rm4sBqCcgRrHzEl6IiLGbf9C0hHb/4qIE4V3A1BI5Rk/IqYj\nYrzz799JmpC0pvRiAMpZ0mt82+skbZR0qMQyALqjzqW+JMn2VZIOSHosIr69zH/fIWlHg7sBKKRW\n+LYHtRD9vog4eLljImJM0ljneD6kA/SwOu/qW9JeSRMR8XL5lQCUVuc1/mZJD0m62/ZHnX/+UHgv\nAAVVXupHxL8lVX6+F0D/4M49ICHCBxIifCAhwgcSInwgIZ6yi0s888wzRea++OKLReZK0po1ZT4+\nsnPnziJzJWn37t1F5vKUXQCXRfhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBC\nhA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEJFHq+9atWq2LBhQ+Nz77///sZnXvDs\ns88Wm43ytm/fXmTugQMHisyVpOHh4cZnnj17VufPn+fx2gAuRfhAQoQPJET4QEKEDyRE+EBChA8k\nVDt828ttH7X9TsmFAJS3lDP+LkkTpRYB0D21wre9VtJWSXvKrgOgG+qe8V+R9KSk+YK7AOiSyvBt\nb5P0dUQcqThuh+3Dtg/Pzc01tiCA5tU542+WdK/tLyW9Lelu22/++KCIGIuI0YgYHRgYaHhNAE2q\nDD8ino6ItRGxTtIDkt6PiAeLbwagGP4eH0hoSdfkEfGhpA+LbAKgazjjAwkRPpAQ4QMJET6QEOED\nCRV5yu6KFStiaGio8bm33npr4zMvOHJk0RsTf7LJyckic9EdIyMjxWZPTU0VmRsRPGUXwKUIH0iI\n8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjw\ngYQIH0iI8IGEivw869nZWU1PTzc+t9RTSSWehnux1atXF5k7MzNTZG5J/fiU3To44wMJET6QEOED\nCRE+kBDhAwkRPpAQ4QMJ1Qrf9jW299v+xPaE7TtKLwagnLo38Lwq6d2I+KPtFZJWFtwJQGGV4du+\nWtJdkv4sSRFxXtL5smsBKKnOpf56SWckvWH7qO09tlcV3gtAQXXCH5C0SdLrEbFR0jlJT/34INs7\nbB+2fbjhHQE0rE74pyWdjohDnV/v18IfBD8QEWMRMRoRo00uCKB5leFHxIykr2zf2PnSFkknim4F\noKi67+o/Imlf5x39zyU9XG4lAKXVCj8iPpLEJTzwM8Gde0BChA8kRPhAQoQPJET4QEKEDyTkiGh+\nqN380D41PDxcbHaJR5jjh0o+XrvE98bExITOnTvnquM44wMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJ\nET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCdX9ablLMjg4\nqKGhocbnHjt2rPGZF2zdurXI3BK/DxeUesruxo0bi8zdu3dvkbklTU1NFZu9bdu2xmeePHmy1nGc\n8YGECB9IiPCBhAgfSIjwgYQIH0iI8IGEaoVv+3Hbx21/bPst21eUXgxAOZXh214j6VFJoxFxs6Tl\nkh4ovRiAcupe6g9IutL2gKSVksrdzgSguMrwI2JS0kuSTkmalvRNRLxXejEA5dS51L9W0n2S1ksa\nkbTK9oOXOW6H7cO2D8/Pzze/KYDG1LnUv0fSFxFxJiJmJR2UdOePD4qIsYgYjYjRZcv4ywKgl9Up\n9JSk222vtG1JWyRNlF0LQEl1XuMfkrRf0rikY53/Z6zwXgAKqvV5/Ih4XtLzhXcB0CW8GAcSInwg\nIcIHEiJ8ICHCBxIifCChIo/Xnp2dLfLo51tuuaXxmReUeozyc889V2SuJL3wwgtF5l5//fVF5pZ0\n2223FZn7/fffF5krSWNj7d0OwxkfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9I\niPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0jIEdH8UPuMpJM1Dx+SdLbxJcrpt30l\ndu6GXtn3VxHxy6qDioS/FLYPR8Roq0ssQb/tK7FzN/TbvlzqAwkRPpBQL4Tf3o8T+Wn6bV+Jnbuh\nr/Zt/TU+gO7rhTM+gC4jfCAhwgcSInwgIcIHEvofNUSAm+o1BD4AAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "row_sums = np.sum(cfm, axis=1)\n", + "err_matrix = cfm / row_sums\n", + "np.fill_diagonal(err_matrix, 0)\n", + "\n", + "plt.matshow(err_matrix, cmap=plt.cm.gray)\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}