Chapter 8 Feature selection for machine learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Univariate Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import read_csv\n",
    "from numpy import set_printoptions\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 111.52  1411.887   17.605   53.108 2175.565  127.669    5.393  181.304]\n",
      "[[148.    0.   33.6  50. ]\n",
      " [ 85.    0.   26.6  31. ]\n",
      " [183.    0.   23.3  32. ]\n",
      " [ 89.   94.   28.1  21. ]\n",
      " [137.  168.   43.1  33. ]]\n"
     ]
    }
   ],
   "source": [
    "filename = 'pima-indians-diabetes.data.csv'\n",
    "names = ['preg','plas','pres','skin','test','mass','pedi','age','class']\n",
    "dataframe = read_csv(filename, names = names)\n",
    "array = dataframe.values\n",
    "X = array[:,0:8]\n",
    "Y = array[:,8]\n",
    "test = SelectKBest(score_func = chi2, k = 4)\n",
    "fit = test.fit(X,Y)\n",
    "set_printoptions(precision=3)\n",
    "print(fit.scores_)\n",
    "features = fit.transform(X)\n",
    "print(features[0:5,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recursive feature elimination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvind/.local/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "model = LogisticRegression()\n",
    "rfe = RFE(model,3)\n",
    "fit = rfe.fit(X,Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num features 3\n",
      "Selected Features [ True False False False False  True  True False]\n",
      "Feature Ranking [1 2 4 5 6 1 1 3]\n"
     ]
    }
   ],
   "source": [
    "print(\"Num features %d\" %fit.n_features_)\n",
    "print(\"Selected Features %s\" % fit.support_)\n",
    "print(\"Feature Ranking %s\" % fit.ranking_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Principal component analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explained Variance: [0.889 0.062 0.026]\n",
      "[[-2.022e-03  9.781e-02  1.609e-02  6.076e-02  9.931e-01  1.401e-02\n",
      "   5.372e-04 -3.565e-03]\n",
      " [-2.265e-02 -9.722e-01 -1.419e-01  5.786e-02  9.463e-02 -4.697e-02\n",
      "  -8.168e-04 -1.402e-01]\n",
      " [-2.246e-02  1.434e-01 -9.225e-01 -3.070e-01  2.098e-02 -1.324e-01\n",
      "  -6.400e-04 -1.255e-01]]\n"
     ]
    }
   ],
   "source": [
    "pca = PCA(n_components = 3)\n",
    "fit = pca.fit(X)\n",
    "print(\"Explained Variance: %s\" % fit.explained_variance_ratio_)\n",
    "print(fit.components_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.112 0.23  0.099 0.078 0.076 0.14  0.119 0.145]\n"
     ]
    }
   ],
   "source": [
    "model = ExtraTreesClassifier()\n",
    "model.fit(X,Y)\n",
    "print(model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}