Python_2_8.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    "# Table of Contents\n",
    " <p><div class=\"lev1 toc-item\"><a href=\"#Datenanalyse-4:-Maschinelles-Lernen\" data-toc-modified-id=\"Datenanalyse-4:-Maschinelles-Lernen-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Datenanalyse 4: Maschinelles Lernen</a></div><div class=\"lev1 toc-item\"><a href=\"#An-introduction-to-machine-learning-with-scikit-learn\" data-toc-modified-id=\"An-introduction-to-machine-learning-with-scikit-learn-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>An introduction to machine learning with scikit-learn</a></div><div class=\"lev1 toc-item\"><a href=\"#A-tutorial-on-statistical-learning-for-scientific-data-processing\" data-toc-modified-id=\"A-tutorial-on-statistical-learning-for-scientific-data-processing-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>A tutorial on statistical-learning for scientific data processing</a></div><div class=\"lev3 toc-item\"><a href=\"#Nearest-Neighbour\" data-toc-modified-id=\"Nearest-Neighbour-301\"><span class=\"toc-item-num\">3.0.1&nbsp;&nbsp;</span>Nearest Neighbour</a></div><div class=\"lev3 toc-item\"><a href=\"#Model-selection\" data-toc-modified-id=\"Model-selection-302\"><span class=\"toc-item-num\">3.0.2&nbsp;&nbsp;</span>Model selection</a></div><div class=\"lev3 toc-item\"><a href=\"#Unsupervised-Clustering\" data-toc-modified-id=\"Unsupervised-Clustering-303\"><span class=\"toc-item-num\">3.0.3&nbsp;&nbsp;</span>Unsupervised Clustering</a></div><div class=\"lev2 toc-item\"><a href=\"#Maschinelles-Lernen-mit-Text\" data-toc-modified-id=\"Maschinelles-Lernen-mit-Text-31\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Maschinelles Lernen mit Text</a></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datenanalyse 4: Maschinelles Lernen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Das Folgende sind Ausschnitte aus dem <a href=\"http://scikit-learn.org/stable/tutorial/index.html\">Tutorial für scikit-learn</a>."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## An introduction to machine learning with scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "digits = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.4,  3.7,  1.5,  0.2])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.data[10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.   0.   5. ...,   0.   0.   0.]\n",
      " [  0.   0.   0. ...,  10.   0.   0.]\n",
      " [  0.   0.   0. ...,  16.   9.   0.]\n",
      " ..., \n",
      " [  0.   0.   1. ...,   6.   0.   0.]\n",
      " [  0.   0.   2. ...,  12.   0.   0.]\n",
      " [  0.   0.  10. ...,  12.   1.   0.]]\n"
     ]
    }
   ],
   "source": [
    "print(digits.data)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2, ..., 8, 9, 8])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.,   0.,   5.,  13.,   9.,   1.,   0.,   0.],\n",
       "       [  0.,   0.,  13.,  15.,  10.,  15.,   5.,   0.],\n",
       "       [  0.,   3.,  15.,   2.,   0.,  11.,   8.,   0.],\n",
       "       [  0.,   4.,  12.,   0.,   0.,   8.,   8.,   0.],\n",
       "       [  0.,   5.,   8.,   0.,   0.,   9.,   8.,   0.],\n",
       "       [  0.,   4.,  11.,   0.,   1.,  12.,   7.,   0.],\n",
       "       [  0.,   2.,  14.,   5.,  10.,  12.,   0.,   0.],\n",
       "       [  0.,   0.,   6.,  13.,  10.,   0.,   0.,   0.]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "digits.images[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "clf = svm.SVC(gamma=0.001, C=100.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None,\n",
       "       estimator=SVC(C=1, cache_size=200, class_weight=None, coef0=0.0, degree=3, gamma=0.0,\n",
       "  kernel='linear', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params={}, iid=True, loss_func=None, n_jobs=-1,\n",
       "       param_grid={'gamma': array([  1.00000e-06,   3.59381e-06,   1.29155e-05,   4.64159e-05,\n",
       "         1.66810e-04,   5.99484e-04,   2.15443e-03,   7.74264e-03,\n",
       "         2.78256e-02,   1.00000e-01])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, score_func=None, scoring=None,\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(digits.data[:-1], digits.target[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([8])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.predict(digits.data[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, degree=3, gamma=0.0,\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "from sklearn import datasets\n",
    "clf = svm.SVC()\n",
    "iris = datasets.load_iris()\n",
    "X, y = iris.data, iris.target\n",
    "clf.fit(X, y)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "s = pickle.dumps(clf)\n",
    "clf2 = pickle.loads(s)\n",
    "clf2.predict(X[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['filename.pkl',\n",
       " 'filename.pkl_01.npy',\n",
       " 'filename.pkl_02.npy',\n",
       " 'filename.pkl_03.npy',\n",
       " 'filename.pkl_04.npy',\n",
       " 'filename.pkl_05.npy',\n",
       " 'filename.pkl_06.npy',\n",
       " 'filename.pkl_07.npy',\n",
       " 'filename.pkl_08.npy',\n",
       " 'filename.pkl_09.npy',\n",
       " 'filename.pkl_10.npy']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "joblib.dump(clf, 'filename.pkl') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    " clf = joblib.load('filename.pkl') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A tutorial on statistical-learning for scientific data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 4)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "data = iris.data\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "digits = datasets.load_digits()\n",
    "digits.images.shape\n",
    "import pylab as pl \n",
    "pl.imshow(digits.images[-1], cmap=pl.cm.gray_r) \n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = digits.images.reshape((digits.images.shape[0], -1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Nearest Neighbour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "iris_X = iris.data\n",
    "iris_y = iris.target\n",
    "np.unique(iris_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_neighbors=5, p=2, weights='uniform')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split iris data in train and test data\n",
    "# A random permutation, to split the data randomly\n",
    "np.random.seed(0)\n",
    "indices = np.random.permutation(len(iris_X))\n",
    "iris_X_train = iris_X[indices[:-10]]\n",
    "iris_y_train = iris_y[indices[:-10]]\n",
    "iris_X_test  = iris_X[indices[-10:]]\n",
    "iris_y_test  = iris_y[indices[-10:]]\n",
    "# Create and fit a nearest-neighbor classifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn = KNeighborsClassifier()\n",
    "knn.fit(iris_X_train, iris_y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 1, 0, 0, 0, 2, 1, 2, 0])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn.predict(iris_X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 0, 0, 0, 2, 1, 2, 0])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris_y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "diabetes = datasets.load_diabetes()\n",
    "diabetes_X_train = diabetes.data[:-20]\n",
    "diabetes_X_test  = diabetes.data[-20:]\n",
    "diabetes_y_train = diabetes.target[:-20]\n",
    "diabetes_y_test  = diabetes.target[-20:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, normalize=False)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "regr = linear_model.LinearRegression()\n",
    "regr.fit(diabetes_X_train, diabetes_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  3.03499549e-01  -2.37639315e+02   5.10530605e+02   3.27736980e+02\n",
      "  -8.14131709e+02   4.92814588e+02   1.02848452e+02   1.84606489e+02\n",
      "   7.43519617e+02   7.60951722e+01]\n"
     ]
    }
   ],
   "source": [
    "print(regr.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2004.5676026898204"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean((regr.predict(diabetes_X_test)-diabetes_y_test)**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.58507530226905757"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr.score(diabetes_X_test, diabetes_y_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = np.c_[ .5, 1].T\n",
    "y = [.5, 1]\n",
    "test = np.c_[ 0, 2].T\n",
    "regr = linear_model.LinearRegression()\n",
    "\n",
    "import pylab as pl \n",
    "pl.figure() \n",
    "\n",
    "np.random.seed(0)\n",
    "for _ in range(6): \n",
    "   this_X = .1*np.random.normal(size=(2, 1)) + X\n",
    "   regr.fit(this_X, y)\n",
    "   pl.plot(test, regr.predict(test)) \n",
    "   pl.scatter(this_X, y, s=3)  \n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "regr = linear_model.Ridge(alpha=.1)\n",
    "\n",
    "pl.figure() \n",
    "\n",
    "np.random.seed(0)\n",
    "for _ in range(6): \n",
    "   this_X = .1*np.random.normal(size=(2, 1)) + X\n",
    "   regr.fit(this_X, y)\n",
    "   pl.plot(test, regr.predict(test)) \n",
    "   pl.scatter(this_X, y, s=3) \n",
    "pl.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.58511106838835292, 0.58520730154446743, 0.58546775406984919, 0.58555120365039159, 0.58307170855541623, 0.570589994372801]\n"
     ]
    }
   ],
   "source": [
    "alphas = np.logspace(-4, -1, 6)\n",
    "from __future__ import print_function\n",
    "print([regr.set_params(alpha=alpha\n",
    "            ).fit(diabetes_X_train, diabetes_y_train,\n",
    "            ).score(diabetes_X_test, diabetes_y_test) for alpha in alphas]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Lasso(alpha=0.025118864315095794, copy_X=True, fit_intercept=True,\n",
       "   max_iter=1000, normalize=False, positive=False, precompute='auto',\n",
       "   tol=0.0001, warm_start=False)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr = linear_model.Lasso()\n",
    "scores = [regr.set_params(alpha=alpha\n",
    "            ).fit(diabetes_X_train, diabetes_y_train\n",
    "            ).score(diabetes_X_test, diabetes_y_test)\n",
    "       for alpha in alphas]\n",
    "best_alpha = alphas[scores.index(max(scores))]\n",
    "regr.alpha = best_alpha\n",
    "regr.fit(diabetes_X_train, diabetes_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   0.         -212.43764548  517.19478111  313.77959962 -160.8303982    -0.\n",
      " -187.19554705   69.38229038  508.66011217   71.84239008]\n"
     ]
    }
   ],
   "source": [
    "print(regr.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logistic = linear_model.LogisticRegression(C=1e5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=100000.0, class_weight=None, dual=False,\n",
       "          fit_intercept=True, intercept_scaling=1, penalty='l2',\n",
       "          random_state=None, tol=0.0001)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic.fit(iris_X_train, iris_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0, degree=3, gamma=0.0,\n",
       "  kernel='linear', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "svc = svm.SVC(kernel='linear')\n",
    "svc.fit(iris_X_train, iris_y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "svc = svm.SVC(kernel='rbf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Model selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97999999999999998"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets, svm\n",
    "digits = datasets.load_digits()\n",
    "X_digits = digits.data\n",
    "y_digits = digits.target\n",
    "svc = svm.SVC(C=1, kernel='linear')\n",
    "svc.fit(X_digits[:-100], y_digits[:-100]).score(X_digits[-100:], y_digits[-100:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.93489148580968284, 0.95659432387312182, 0.93989983305509184]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "X_folds = np.array_split(X_digits, 3)\n",
    "y_folds = np.array_split(y_digits, 3)\n",
    "scores = list()\n",
    "for k in range(3):\n",
    "    # We use 'list' to copy, in order to 'pop' later on\n",
    "    X_train = list(X_folds)\n",
    "    X_test  = X_train.pop(k)\n",
    "    X_train = np.concatenate(X_train)\n",
    "    y_train = list(y_folds)\n",
    "    y_test  = y_train.pop(k)\n",
    "    y_train = np.concatenate(y_train)\n",
    "    scores.append(svc.fit(X_train, y_train).score(X_test, y_test))\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: [2 3 4 5] | test: [0 1]\n",
      "Train: [0 1 4 5] | test: [2 3]\n",
      "Train: [0 1 2 3] | test: [4 5]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import cross_validation\n",
    "k_fold = cross_validation.KFold(n=6, n_folds=3)\n",
    "for train_indices, test_indices in k_fold:\n",
    "     print('Train: %s | test: %s' % (train_indices, test_indices))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.93489148580968284, 0.95659432387312182, 0.93989983305509184]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> kfold = cross_validation.KFold(len(X_digits), n_folds=3)\n",
    ">>> [svc.fit(X_digits[train], y_digits[train]).score(X_digits[test], y_digits[test])\n",
    "...          for train, test in kfold]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.93489149,  0.95659432,  0.93989983])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validation.cross_val_score(svc, X_digits, y_digits, cv=kfold, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None,\n",
       "       estimator=SVC(C=1, cache_size=200, class_weight=None, coef0=0.0, degree=3, gamma=0.0,\n",
       "  kernel='linear', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params={}, iid=True, loss_func=None, n_jobs=-1,\n",
       "       param_grid={'gamma': array([  1.00000e-06,   3.59381e-06,   1.29155e-05,   4.64159e-05,\n",
       "         1.66810e-04,   5.99484e-04,   2.15443e-03,   7.74264e-03,\n",
       "         2.78256e-02,   1.00000e-01])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, score_func=None, scoring=None,\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> from sklearn.grid_search import GridSearchCV\n",
    ">>> gammas = np.logspace(-6, -1, 10)\n",
    ">>> clf = GridSearchCV(estimator=svc, param_grid=dict(gamma=gammas),\n",
    "...                    n_jobs=-1)\n",
    ">>> clf.fit(X_digits[:1000], y_digits[:1000])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.92400000000000004"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> clf.best_score_  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_estimator_.gamma == 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94228356336260977"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> # Prediction performance on test set is not as good as on train set\n",
    ">>> clf.score(X_digits[1000:], y_digits[1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.93521595,  0.95826377,  0.93791946])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validation.cross_val_score(clf, X_digits, y_digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LassoCV(alphas=None, copy_X=True, cv=None, eps=0.001, fit_intercept=True,\n",
       "    max_iter=1000, n_alphas=100, n_jobs=1, normalize=False, positive=False,\n",
       "    precompute='auto', tol=0.0001, verbose=False)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> from sklearn import linear_model, datasets\n",
    ">>> lasso = linear_model.LassoCV()\n",
    ">>> diabetes = datasets.load_diabetes()\n",
    ">>> X_diabetes = diabetes.data\n",
    ">>> y_diabetes = diabetes.target\n",
    ">>> lasso.fit(X_diabetes, y_diabetes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.012291895087486173"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> # The estimator chose automatically its lambda:\n",
    ">>> lasso.alpha_ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Unsupervised Clustering "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(copy_x=True, init='k-means++', max_iter=300, n_clusters=3, n_init=10,\n",
       "    n_jobs=1, precompute_distances=True, random_state=None, tol=0.0001,\n",
       "    verbose=0)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> from sklearn import cluster, datasets\n",
    ">>> iris = datasets.load_iris()\n",
    ">>> X_iris = iris.data\n",
    ">>> y_iris = iris.target\n",
    "\n",
    ">>> k_means = cluster.KMeans(n_clusters=3)\n",
    ">>> k_means.fit(X_iris) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 1 1 0 0 0 0 0 2 2 2 2 2]\n"
     ]
    }
   ],
   "source": [
    "print(k_means.labels_[::10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 1 1 1 1 1 2 2 2 2 2]\n"
     ]
    }
   ],
   "source": [
    "print(y_iris[::10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(copy_x=True, init='k-means++', max_iter=300, n_clusters=5, n_init=1,\n",
       "    n_jobs=1, precompute_distances=True, random_state=None, tol=0.0001,\n",
       "    verbose=0)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> import scipy as sp\n",
    ">>> try:\n",
    "...    lena = sp.lena()\n",
    "... except AttributeError:\n",
    "...    from scipy import misc\n",
    "...    lena = misc.lena()\n",
    ">>> X = lena.reshape((-1, 1)) # We need an (n_sample, n_feature) array\n",
    ">>> k_means = cluster.KMeans(n_clusters=5, n_init=1)\n",
    ">>> k_means.fit(X) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    ">>> values = k_means.cluster_centers_.squeeze()\n",
    ">>> labels = k_means.labels_\n",
    ">>> lena_compressed = np.choose(labels, values)\n",
    ">>> lena_compressed.shape = lena.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compute structured hierarchical clustering...\n",
      "Number of pixels:  65536\n",
      "Number of clusters:  15\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.image import grid_to_graph\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "\n",
    "###############################################################################\n",
    "# Generate data\n",
    "lena = sp.misc.lena()\n",
    "# Downsample the image by a factor of 4\n",
    "lena = lena[::2, ::2] + lena[1::2, ::2] + lena[::2, 1::2] + lena[1::2, 1::2]\n",
    "X = np.reshape(lena, (-1, 1))\n",
    "\n",
    "###############################################################################\n",
    "# Define the structure A of the data. Pixels connected to their neighbors.\n",
    "connectivity = grid_to_graph(*lena.shape)\n",
    "\n",
    "###############################################################################\n",
    "# Compute clustering\n",
    "print(\"Compute structured hierarchical clustering...\")\n",
    "\n",
    "n_clusters = 15  # number of regions\n",
    "ward = AgglomerativeClustering(n_clusters=n_clusters,\n",
    "        linkage='ward', connectivity=connectivity).fit(X)\n",
    "label = np.reshape(ward.labels_, lena.shape)\n",
    "print(\"Number of pixels: \", label.size)\n",
    "print(\"Number of clusters: \", np.unique(label).size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FeatureAgglomeration(affinity='euclidean', compute_full_tree='auto',\n",
       "           connectivity=<64x64 sparse matrix of type '<class 'numpy.int32'>'\n",
       "\twith 288 stored elements in COOrdinate format>,\n",
       "           linkage='ward', memory=Memory(cachedir=None), n_clusters=32,\n",
       "           n_components=None, pooling_func=<function mean at 0x03CEC540>)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> digits = datasets.load_digits()\n",
    ">>> images = digits.images\n",
    ">>> X = np.reshape(images, (len(images), -1))\n",
    ">>> connectivity = grid_to_graph(*images[0].shape)\n",
    "\n",
    ">>> agglo = cluster.FeatureAgglomeration(connectivity=connectivity,\n",
    "...                                      n_clusters=32)\n",
    ">>> agglo.fit(X) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    ">>> X_reduced = agglo.transform(X)\n",
    "\n",
    ">>> X_approx = agglo.inverse_transform(X_reduced)\n",
    ">>> images_approx = np.reshape(X_approx, images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA(copy=True, n_components=None, whiten=False)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> # Create a signal with only 2 useful dimensions\n",
    ">>> x1 = np.random.normal(size=100)\n",
    ">>> x2 = np.random.normal(size=100)\n",
    ">>> x3 = x1 + x2\n",
    ">>> X = np.c_[x1, x2, x3]\n",
    "\n",
    ">>> from sklearn import decomposition\n",
    ">>> pca = decomposition.PCA()\n",
    ">>> pca.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  3.29154194e+00   8.57774851e-01   1.09332636e-32]\n"
     ]
    }
   ],
   "source": [
    "print(pca.explained_variance_)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 2)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# As we can see, only the 2 first components are useful\n",
    ">>> pca.n_components = 2\n",
    ">>> X_reduced = pca.fit_transform(X)\n",
    ">>> X_reduced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> # Generate sample data\n",
    ">>> time = np.linspace(0, 10, 2000)\n",
    ">>> s1 = np.sin(2 * time)  # Signal 1 : sinusoidal signal\n",
    ">>> s2 = np.sign(np.sin(3 * time))  # Signal 2 : square signal\n",
    ">>> S = np.c_[s1, s2]\n",
    ">>> S += 0.2 * np.random.normal(size=S.shape)  # Add noise\n",
    ">>> S /= S.std(axis=0)  # Standardize data\n",
    ">>> # Mix data\n",
    ">>> A = np.array([[1, 1], [0.5, 2]])  # Mixing matrix\n",
    ">>> X = np.dot(S, A.T)  # Generate observations\n",
    "\n",
    ">>> # Compute ICA\n",
    ">>> ica = decomposition.FastICA()\n",
    ">>> S_ = ica.fit_transform(X)  # Get the estimated sources\n",
    ">>> A_ = ica.mixing_.T\n",
    ">>> np.allclose(X,  np.dot(S_, A_) + ica.mean_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Maschinelles Lernen mit Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    " categories = ['alt.atheism', 'soc.religion.christian',\n",
    "               'comp.graphics', 'sci.med']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python34\\lib\\site-packages\\sklearn\\datasets\\twenty_newsgroups.py:89: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead\n",
      "  logger.warn(\"Downloading dataset from %s (14 MB)\", URL)\n",
      "WARNING:sklearn.datasets.twenty_newsgroups:Downloading dataset from http://people.csail.mit.edu/jrennie/20Newsgroups/20news-bydate.tar.gz (14 MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "________________________________________________________________________________\n",
      "Cache loading failed\n",
      "________________________________________________________________________________\n",
      "decoding with 'zlib_codec' codec failed (error: Error -5 while decompressing data: incomplete or truncated stream)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "twenty_train = fetch_20newsgroups(subset='train',\n",
    "    categories=categories, shuffle=True, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alt.atheism', 'comp.graphics', 'sci.med', 'soc.religion.christian']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twenty_train.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2257"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(twenty_train.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2257"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(twenty_train.filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "comp.graphics\n"
     ]
    }
   ],
   "source": [
    "print(twenty_train.target_names[twenty_train.target[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 3, 3, 3, 3, 3, 2, 2, 2], dtype=int32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twenty_train.target[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "comp.graphics\n",
      "comp.graphics\n",
      "soc.religion.christian\n",
      "soc.religion.christian\n",
      "soc.religion.christian\n",
      "soc.religion.christian\n",
      "soc.religion.christian\n",
      "sci.med\n",
      "sci.med\n",
      "sci.med\n"
     ]
    }
   ],
   "source": [
    "for t in twenty_train.target[:10]:\n",
    "    print(twenty_train.target_names[t])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2257, 35788)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "count_vect = CountVectorizer()\n",
    "X_train_counts = count_vect.fit_transform(twenty_train.data)\n",
    "X_train_counts.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4690"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vect.vocabulary_.get(u'algorithm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2257, 35788)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)\n",
    "X_train_tf = tfidf_transformer.transform(X_train_counts)\n",
    "X_train_tf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "clf = MultinomialNB().fit(X_train_tf, twenty_train.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'God is love' => soc.religion.christian\n",
      "'OpenGL on the GPU is fast' => comp.graphics\n"
     ]
    }
   ],
   "source": [
    "docs_new = ['God is love', 'OpenGL on the GPU is fast']\n",
    "X_new_counts = count_vect.transform(docs_new)\n",
    "X_new_tfidf = tfidf_transformer.transform(X_new_counts)\n",
    "\n",
    "predicted = clf.predict(X_new_tfidf)\n",
    "\n",
    "for doc, category in zip(docs_new, predicted):\n",
    "    print('%r => %s' % (doc, twenty_train.target_names[category]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "text_clf = Pipeline([('vect', CountVectorizer()),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', MultinomialNB()),])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "text_clf = text_clf.fit(twenty_train.data, twenty_train.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83488681757656458"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "twenty_test = fetch_20newsgroups(subset='test',\n",
    "    categories=categories, shuffle=True, random_state=42)\n",
    "docs_test = twenty_test.data\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == twenty_test.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9127829560585885"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "text_clf = Pipeline([('vect', CountVectorizer()),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                                           alpha=1e-3, n_iter=5)),])\n",
    "_ = text_clf.fit(twenty_train.data, twenty_train.target)\n",
    "predicted = text_clf.predict(docs_test)\n",
    "np.mean(predicted == twenty_test.target)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        precision    recall  f1-score   support\n",
      "\n",
      "           alt.atheism       0.94      0.82      0.87       319\n",
      "         comp.graphics       0.88      0.98      0.92       389\n",
      "               sci.med       0.95      0.89      0.92       396\n",
      "soc.religion.christian       0.90      0.95      0.92       398\n",
      "\n",
      "           avg / total       0.92      0.91      0.91      1502\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print(metrics.classification_report(twenty_test.target, predicted,\n",
    "    target_names=twenty_test.target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2)],\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'clf__alpha': (1e-2, 1e-3),}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gs_clf = gs_clf.fit(twenty_train.data[:400], twenty_train.target[:400])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'soc.religion.christian'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twenty_train.target_names[gs_clf.predict(['God is love'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf__alpha: 0.001\n",
      "tfidf__use_idf: True\n",
      "vect__ngram_range: (1, 1)\n"
     ]
    }
   ],
   "source": [
    "best_parameters, score, _ = max(gs_clf.grid_scores_, key=lambda x: x[1])\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90249999999999997"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score                                              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "157px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": true,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}