Z05_C_II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Regresja"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn import datasets, linear_model\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn import model_selection\n",
    "from sklearn import  metrics\n",
    "from statsmodels.formula.api import ols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rozważmy zbiór Boston"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "data_url = \"http://lib.stat.cmu.edu/datasets/boston\"\n",
    "raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None)\n",
    "data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])\n",
    "target = raw_df.values[1::2, 2]\n",
    "feature_names = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>PRICE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  PRICE  \n",
       "0     15.3  396.90   4.98   24.0  \n",
       "1     17.8  396.90   9.14   21.6  \n",
       "2     17.8  392.83   4.03   34.7  \n",
       "3     18.7  394.63   2.94   33.4  \n",
       "4     18.7  396.90   5.33   36.2  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print(boston.keys())\n",
    "# print(boston.data.shape)\n",
    "# print(boston.feature_names)\n",
    "bos=pd.DataFrame(data)\n",
    "bos.head()\n",
    "\n",
    "bos.columns = feature_names\n",
    "bos.head()\n",
    "\n",
    "# print(boston.target[0:5])\n",
    "bos['PRICE']=target\n",
    "bos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the data\n",
    "boston_X = data\n",
    "boston_Y = target\n",
    "\n",
    "# Split the data into training/testing sets\n",
    "boston_X_train = boston_X[:-50]\n",
    "boston_X_test = boston_X[-50:]\n",
    " \n",
    "# Split the targets into training/testing sets\n",
    "boston_y_train = boston_Y[:-50]\n",
    "boston_y_test = boston_Y[-50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>PRICE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  PRICE  \n",
       "0     15.3  396.90   4.98   24.0  \n",
       "1     17.8  396.90   9.14   21.6  \n",
       "2     17.8  392.83   4.03   34.7  \n",
       "3     18.7  394.63   2.94   33.4  \n",
       "4     18.7  396.90   5.33   36.2  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bos=pd.DataFrame(data[:-50])\n",
    "bos.columns = feature_names\n",
    "bos['PRICE']=target[:-50]\n",
    "bos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>PRICE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.66883</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713</td>\n",
       "      <td>5.976</td>\n",
       "      <td>87.9</td>\n",
       "      <td>2.5806</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>10.48</td>\n",
       "      <td>19.01</td>\n",
       "      <td>12.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8.20058</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713</td>\n",
       "      <td>5.936</td>\n",
       "      <td>80.3</td>\n",
       "      <td>2.7792</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>3.50</td>\n",
       "      <td>16.94</td>\n",
       "      <td>13.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.75223</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713</td>\n",
       "      <td>6.301</td>\n",
       "      <td>83.7</td>\n",
       "      <td>2.7831</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>272.21</td>\n",
       "      <td>16.23</td>\n",
       "      <td>14.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.80117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713</td>\n",
       "      <td>6.081</td>\n",
       "      <td>84.4</td>\n",
       "      <td>2.7175</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>396.90</td>\n",
       "      <td>14.70</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.81213</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713</td>\n",
       "      <td>6.701</td>\n",
       "      <td>90.0</td>\n",
       "      <td>2.5975</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>255.23</td>\n",
       "      <td>16.42</td>\n",
       "      <td>16.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM   ZN  INDUS  CHAS    NOX     RM   AGE     DIS   RAD    TAX  \\\n",
       "0  4.66883  0.0   18.1   0.0  0.713  5.976  87.9  2.5806  24.0  666.0   \n",
       "1  8.20058  0.0   18.1   0.0  0.713  5.936  80.3  2.7792  24.0  666.0   \n",
       "2  7.75223  0.0   18.1   0.0  0.713  6.301  83.7  2.7831  24.0  666.0   \n",
       "3  6.80117  0.0   18.1   0.0  0.713  6.081  84.4  2.7175  24.0  666.0   \n",
       "4  4.81213  0.0   18.1   0.0  0.713  6.701  90.0  2.5975  24.0  666.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  PRICE  \n",
       "0     20.2   10.48  19.01   12.7  \n",
       "1     20.2    3.50  16.94   13.5  \n",
       "2     20.2  272.21  16.23   14.9  \n",
       "3     20.2  396.90  14.70   20.0  \n",
       "4     20.2  255.23  16.42   16.4  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bos1=pd.DataFrame(boston_X[-50:])\n",
    "bos1.columns = feature_names\n",
    "bos1['PRICE']=target[-50:]\n",
    "bos1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Results: Ordinary least squares\n",
      "==================================================================\n",
      "Model:              OLS              Adj. R-squared:     0.155    \n",
      "Dependent Variable: PRICE            AIC:                3271.6219\n",
      "Date:               2023-04-03 13:06 BIC:                3279.8669\n",
      "No. Observations:   456              Log-Likelihood:     -1633.8  \n",
      "Df Model:           1                F-statistic:        84.65    \n",
      "Df Residuals:       454              Prob (F-statistic): 1.30e-18 \n",
      "R-squared:          0.157            Scale:              76.123   \n",
      "-------------------------------------------------------------------\n",
      "                Coef.   Std.Err.     t     P>|t|    [0.025   0.975]\n",
      "-------------------------------------------------------------------\n",
      "Intercept      24.4313    0.4395  55.5870  0.0000  23.5676  25.2951\n",
      "I(CRIM)        -0.4202    0.0457  -9.2008  0.0000  -0.5100  -0.3305\n",
      "------------------------------------------------------------------\n",
      "Omnibus:             117.949       Durbin-Watson:          0.726  \n",
      "Prob(Omnibus):       0.000         Jarque-Bera (JB):       226.399\n",
      "Skew:                1.446         Prob(JB):               0.000  \n",
      "Kurtosis:            4.884         Condition No.:          10     \n",
      "==================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = ols(\"PRICE ~ I(CRIM)\", bos).fit()\n",
    "# Print the summary\n",
    "print((model.summary2()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.15715876242331683"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import  metrics\n",
    "metrics.r2_score(bos.PRICE, model.predict(bos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.8973506326938079"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import  metrics\n",
    "metrics.r2_score(bos1.PRICE, model.predict(bos1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}