Problem Set 4-ML-Preethi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem Set 4- ML -Preethi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading the packages\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from scipy import stats\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "import matplotlib as mpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reading the data\n",
    "df = pd.read_csv('C:/Users/Preethi Abraham/Desktop/Brandeis Studies/Machine Learning and Data Analytics/KaggleAmes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop a bunch of useless series\n",
    "bigDrop = False\n",
    "if (bigDrop):\n",
    "    df.drop(df.columns[[0,2,6, 7, 8,9,10,11,12,13,14,21,22,23,24,25,27,28,29,\n",
    "                        32,33,35,36,39,40,42,45,47,48,49,50,52,53,55,57,60,63,\n",
    "                        64,65,68,69,70,71,72,73,74,75,76,78,79]],axis=1,inplace=True)\n",
    "if(not(bigDrop)):\n",
    "    df.drop(df.columns[[0]],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        MSSubClass  LotFrontage        LotArea  OverallQual  OverallCond  \\\n",
      "count  1460.000000  1201.000000    1460.000000  1460.000000  1460.000000   \n",
      "mean     56.897260    70.049958   10516.828082     6.099315     5.575342   \n",
      "std      42.300571    24.284752    9981.264932     1.382997     1.112799   \n",
      "min      20.000000    21.000000    1300.000000     1.000000     1.000000   \n",
      "25%      20.000000    59.000000    7553.500000     5.000000     5.000000   \n",
      "50%      50.000000    69.000000    9478.500000     6.000000     5.000000   \n",
      "75%      70.000000    80.000000   11601.500000     7.000000     6.000000   \n",
      "max     190.000000   313.000000  215245.000000    10.000000     9.000000   \n",
      "\n",
      "         YearBuilt  YearRemodAdd   MasVnrArea   BsmtFinSF1   BsmtFinSF2  ...  \\\n",
      "count  1460.000000   1460.000000  1452.000000  1460.000000  1460.000000  ...   \n",
      "mean   1971.267808   1984.865753   103.685262   443.639726    46.549315  ...   \n",
      "std      30.202904     20.645407   181.066207   456.098091   161.319273  ...   \n",
      "min    1872.000000   1950.000000     0.000000     0.000000     0.000000  ...   \n",
      "25%    1954.000000   1967.000000     0.000000     0.000000     0.000000  ...   \n",
      "50%    1973.000000   1994.000000     0.000000   383.500000     0.000000  ...   \n",
      "75%    2000.000000   2004.000000   166.000000   712.250000     0.000000  ...   \n",
      "max    2010.000000   2010.000000  1600.000000  5644.000000  1474.000000  ...   \n",
      "\n",
      "        WoodDeckSF  OpenPorchSF  EnclosedPorch    3SsnPorch  ScreenPorch  \\\n",
      "count  1460.000000  1460.000000    1460.000000  1460.000000  1460.000000   \n",
      "mean     94.244521    46.660274      21.954110     3.409589    15.060959   \n",
      "std     125.338794    66.256028      61.119149    29.317331    55.757415   \n",
      "min       0.000000     0.000000       0.000000     0.000000     0.000000   \n",
      "25%       0.000000     0.000000       0.000000     0.000000     0.000000   \n",
      "50%       0.000000    25.000000       0.000000     0.000000     0.000000   \n",
      "75%     168.000000    68.000000       0.000000     0.000000     0.000000   \n",
      "max     857.000000   547.000000     552.000000   508.000000   480.000000   \n",
      "\n",
      "          PoolArea       MiscVal       MoSold       YrSold      SalePrice  \n",
      "count  1460.000000   1460.000000  1460.000000  1460.000000    1460.000000  \n",
      "mean      2.758904     43.489041     6.321918  2007.815753  180921.195890  \n",
      "std      40.177307    496.123024     2.703626     1.328095   79442.502883  \n",
      "min       0.000000      0.000000     1.000000  2006.000000   34900.000000  \n",
      "25%       0.000000      0.000000     5.000000  2007.000000  129975.000000  \n",
      "50%       0.000000      0.000000     6.000000  2008.000000  163000.000000  \n",
      "75%       0.000000      0.000000     8.000000  2009.000000  214000.000000  \n",
      "max     738.000000  15500.000000    12.000000  2010.000000  755000.000000  \n",
      "\n",
      "[8 rows x 37 columns]\n",
      "(1460, 80)\n"
     ]
    }
   ],
   "source": [
    "print(df.describe())\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop series with similar stuff\n",
    "# Shift param down to 0.95\n",
    "# 1.00 = don't drop anything !!!\n",
    "sameThresh = 1.00\n",
    "dropSameList = []\n",
    "for i in range(df.shape[1]):\n",
    "    xsame = np.mean( df.iloc[0,i]==df.iloc[:,i])\n",
    "    if(xsame>sameThresh):\n",
    "        dropSameList.append(i) \n",
    "df.drop(df.columns[[dropSameList]],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check NA's in each column\n",
    "percent_missing = df.isnull().sum() * 100 / len(df)\n",
    "missing_value_df = pd.DataFrame({'column_name': df.columns,\n",
    "                                 'percent_missing': percent_missing})\n",
    "drop_col = missing_value_df[missing_value_df.percent_missing > 50]\n",
    "missing_list = drop_col.column_name.values\n",
    "df.drop(missing_list,axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        MSSubClass  LotFrontage        LotArea  OverallQual  OverallCond  \\\n",
      "count  1460.000000  1460.000000    1460.000000  1460.000000  1460.000000   \n",
      "mean     56.897260    70.049958   10516.828082     6.099315     5.575342   \n",
      "std      42.300571    22.024023    9981.264932     1.382997     1.112799   \n",
      "min      20.000000    21.000000    1300.000000     1.000000     1.000000   \n",
      "25%      20.000000    60.000000    7553.500000     5.000000     5.000000   \n",
      "50%      50.000000    70.049958    9478.500000     6.000000     5.000000   \n",
      "75%      70.000000    79.000000   11601.500000     7.000000     6.000000   \n",
      "max     190.000000   313.000000  215245.000000    10.000000     9.000000   \n",
      "\n",
      "         YearBuilt  YearRemodAdd   MasVnrArea   BsmtFinSF1   BsmtFinSF2  ...  \\\n",
      "count  1460.000000   1460.000000  1460.000000  1460.000000  1460.000000  ...   \n",
      "mean   1971.267808   1984.865753   103.685262   443.639726    46.549315  ...   \n",
      "std      30.202904     20.645407   180.569112   456.098091   161.319273  ...   \n",
      "min    1872.000000   1950.000000     0.000000     0.000000     0.000000  ...   \n",
      "25%    1954.000000   1967.000000     0.000000     0.000000     0.000000  ...   \n",
      "50%    1973.000000   1994.000000     0.000000   383.500000     0.000000  ...   \n",
      "75%    2000.000000   2004.000000   164.250000   712.250000     0.000000  ...   \n",
      "max    2010.000000   2010.000000  1600.000000  5644.000000  1474.000000  ...   \n",
      "\n",
      "       SaleType_ConLw  SaleType_New  SaleType_Oth  SaleType_WD  \\\n",
      "count     1460.000000   1460.000000   1460.000000  1460.000000   \n",
      "mean         0.003425      0.083562      0.002055     0.867808   \n",
      "std          0.058440      0.276824      0.045299     0.338815   \n",
      "min          0.000000      0.000000      0.000000     0.000000   \n",
      "25%          0.000000      0.000000      0.000000     1.000000   \n",
      "50%          0.000000      0.000000      0.000000     1.000000   \n",
      "75%          0.000000      0.000000      0.000000     1.000000   \n",
      "max          1.000000      1.000000      1.000000     1.000000   \n",
      "\n",
      "       SaleCondition_Abnorml  SaleCondition_AdjLand  SaleCondition_Alloca  \\\n",
      "count            1460.000000            1460.000000           1460.000000   \n",
      "mean                0.069178               0.002740              0.008219   \n",
      "std                 0.253844               0.052289              0.090317   \n",
      "min                 0.000000               0.000000              0.000000   \n",
      "25%                 0.000000               0.000000              0.000000   \n",
      "50%                 0.000000               0.000000              0.000000   \n",
      "75%                 0.000000               0.000000              0.000000   \n",
      "max                 1.000000               1.000000              1.000000   \n",
      "\n",
      "       SaleCondition_Family  SaleCondition_Normal  SaleCondition_Partial  \n",
      "count           1460.000000           1460.000000            1460.000000  \n",
      "mean               0.013699              0.820548               0.085616  \n",
      "std                0.116277              0.383862               0.279893  \n",
      "min                0.000000              0.000000               0.000000  \n",
      "25%                0.000000              1.000000               0.000000  \n",
      "50%                0.000000              1.000000               0.000000  \n",
      "75%                0.000000              1.000000               0.000000  \n",
      "max                1.000000              1.000000               1.000000  \n",
      "\n",
      "[8 rows x 276 columns]\n"
     ]
    }
   ],
   "source": [
    "# generate dummies\n",
    "df = pd.get_dummies(df)\n",
    "# fill na's with means\n",
    "df = df.fillna(df.mean())\n",
    "print(df.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#change y to SalesPrice log \n",
    "# y = np.log(df.SalePrice)\n",
    "y = df.SalePrice\n",
    "\n",
    "#drop original dependent variable\n",
    "X = df.drop(['SalePrice'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>...</th>\n",
       "      <th>SaleType_ConLw</th>\n",
       "      <th>SaleType_New</th>\n",
       "      <th>SaleType_Oth</th>\n",
       "      <th>SaleType_WD</th>\n",
       "      <th>SaleCondition_Abnorml</th>\n",
       "      <th>SaleCondition_AdjLand</th>\n",
       "      <th>SaleCondition_Alloca</th>\n",
       "      <th>SaleCondition_Family</th>\n",
       "      <th>SaleCondition_Normal</th>\n",
       "      <th>SaleCondition_Partial</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2003</td>\n",
       "      <td>2003</td>\n",
       "      <td>196.0</td>\n",
       "      <td>706</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1976</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2001</td>\n",
       "      <td>2002</td>\n",
       "      <td>162.0</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1915</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>350.0</td>\n",
       "      <td>655</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 276 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MSSubClass  LotFrontage  LotArea  OverallQual  OverallCond  YearBuilt  \\\n",
       "0          60         65.0     8450            7            5       2003   \n",
       "1          20         80.0     9600            6            8       1976   \n",
       "2          60         68.0    11250            7            5       2001   \n",
       "3          70         60.0     9550            7            5       1915   \n",
       "4          60         84.0    14260            8            5       2000   \n",
       "\n",
       "   YearRemodAdd  MasVnrArea  BsmtFinSF1  BsmtFinSF2  ...  SaleType_ConLw  \\\n",
       "0          2003       196.0         706           0  ...               0   \n",
       "1          1976         0.0         978           0  ...               0   \n",
       "2          2002       162.0         486           0  ...               0   \n",
       "3          1970         0.0         216           0  ...               0   \n",
       "4          2000       350.0         655           0  ...               0   \n",
       "\n",
       "   SaleType_New  SaleType_Oth  SaleType_WD  SaleCondition_Abnorml  \\\n",
       "0             0             0            1                      0   \n",
       "1             0             0            1                      0   \n",
       "2             0             0            1                      0   \n",
       "3             0             0            1                      1   \n",
       "4             0             0            1                      0   \n",
       "\n",
       "   SaleCondition_AdjLand  SaleCondition_Alloca  SaleCondition_Family  \\\n",
       "0                      0                     0                     0   \n",
       "1                      0                     0                     0   \n",
       "2                      0                     0                     0   \n",
       "3                      0                     0                     0   \n",
       "4                      0                     0                     0   \n",
       "\n",
       "   SaleCondition_Normal  SaleCondition_Partial  \n",
       "0                     1                      0  \n",
       "1                     1                      0  \n",
       "2                     1                      0  \n",
       "3                     0                      0  \n",
       "4                     1                      0  \n",
       "\n",
       "[5 rows x 276 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>...</th>\n",
       "      <th>SaleType_ConLw</th>\n",
       "      <th>SaleType_New</th>\n",
       "      <th>SaleType_Oth</th>\n",
       "      <th>SaleType_WD</th>\n",
       "      <th>SaleCondition_Abnorml</th>\n",
       "      <th>SaleCondition_AdjLand</th>\n",
       "      <th>SaleCondition_Alloca</th>\n",
       "      <th>SaleCondition_Family</th>\n",
       "      <th>SaleCondition_Normal</th>\n",
       "      <th>SaleCondition_Partial</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2003</td>\n",
       "      <td>2003</td>\n",
       "      <td>196.0</td>\n",
       "      <td>706</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1976</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2001</td>\n",
       "      <td>2002</td>\n",
       "      <td>162.0</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1915</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>350.0</td>\n",
       "      <td>655</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 275 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MSSubClass  LotFrontage  LotArea  OverallQual  OverallCond  YearBuilt  \\\n",
       "0          60         65.0     8450            7            5       2003   \n",
       "1          20         80.0     9600            6            8       1976   \n",
       "2          60         68.0    11250            7            5       2001   \n",
       "3          70         60.0     9550            7            5       1915   \n",
       "4          60         84.0    14260            8            5       2000   \n",
       "\n",
       "   YearRemodAdd  MasVnrArea  BsmtFinSF1  BsmtFinSF2  ...  SaleType_ConLw  \\\n",
       "0          2003       196.0         706           0  ...               0   \n",
       "1          1976         0.0         978           0  ...               0   \n",
       "2          2002       162.0         486           0  ...               0   \n",
       "3          1970         0.0         216           0  ...               0   \n",
       "4          2000       350.0         655           0  ...               0   \n",
       "\n",
       "   SaleType_New  SaleType_Oth  SaleType_WD  SaleCondition_Abnorml  \\\n",
       "0             0             0            1                      0   \n",
       "1             0             0            1                      0   \n",
       "2             0             0            1                      0   \n",
       "3             0             0            1                      1   \n",
       "4             0             0            1                      0   \n",
       "\n",
       "   SaleCondition_AdjLand  SaleCondition_Alloca  SaleCondition_Family  \\\n",
       "0                      0                     0                     0   \n",
       "1                      0                     0                     0   \n",
       "2                      0                     0                     0   \n",
       "3                      0                     0                     0   \n",
       "4                      0                     0                     0   \n",
       "\n",
       "   SaleCondition_Normal  SaleCondition_Partial  \n",
       "0                     1                      0  \n",
       "1                     1                      0  \n",
       "2                     1                      0  \n",
       "3                     0                      0  \n",
       "4                     1                      0  \n",
       "\n",
       "[5 rows x 275 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this problem you should download the KaggleAmes.csv data set from the data portion of the Latte website.  This is a large data set with house prices and lots of features variables.  Also, download the Jupyter notebook below which loads the data and does some initial preprocessing and cleaning.  It will eventually put the data you should use in the X and y variables as usual. For grid searches use the gridSearchCV() function.\n",
    "1. Run a linear regression on the full sample, and report the R-squared value as you might have done in a standard econometrics class.\n",
    "2. Now run the usual train/test split monte-carlo with half the sample in the test set.  Do this for 100 monte-carlo iterations, and report the mean R-squared on the train and test sets.\n",
    "3. Repeat the same experiment (test/train split monte-carlo) for a small Ridge regression grid search.  Just use alpha = [0.5,1.,2.].  How is it doing compared to regular least squares on test data?\n",
    "4. Repeat this, but this time add the StandardScaler() to your data preprocessing.  Did your results change?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Full sample R-squared:\n",
      "0.9313624499887028\n"
     ]
    }
   ],
   "source": [
    "#1.\n",
    "lr = LinearRegression()\n",
    "lr.fit(X,y)\n",
    "print(\"Full sample R-squared:\")\n",
    "print(lr.score(X,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#2.\n",
    "def MCtraintest(nmc,X,y,modelObj,testFrac):\n",
    "    # make space to store scores\n",
    "    trainScore = np.zeros(nmc)\n",
    "    testScore  = np.zeros(nmc)\n",
    "    for i in range(nmc):\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=testFrac)\n",
    "        modelObj.fit(X_train,y_train)\n",
    "        trainScore[i] = modelObj.score(X_train,y_train)\n",
    "        testScore[i]  = modelObj.score(X_test,y_test)\n",
    "    return trainScore,testScore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Score= 0.9496383835487886\n",
      "Test Score= -21023213.092998773\n"
     ]
    }
   ],
   "source": [
    "trainS, testS = MCtraintest(100,X,y,lr,0.5)\n",
    "print(\"Train Score=\",np.mean(trainS))\n",
    "print(\"Test Score=\",np.mean(testS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   rank_test_score  mean_test_score param_alpha\n",
      "0                3         0.771688         0.5\n",
      "1                2         0.775871           1\n",
      "2                1         0.779921           2\n"
     ]
    }
   ],
   "source": [
    "#3.\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "score_used = 'r2' \n",
    "param_grid={'alpha':[0.5,1.,2.]}\n",
    "shuffle_split = ShuffleSplit(test_size=0.5, n_splits=100)\n",
    "grid_search=GridSearchCV(Ridge(),param_grid,cv=shuffle_split,scoring=score_used, \n",
    "                              return_train_score=True)\n",
    "grid_search.fit(X,y)\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "print(results[['rank_test_score','mean_test_score','param_alpha']])\n",
    "\n",
    "#Compared to the monte carlo simulation scores, the ridge regression seems to be doing much better.\n",
    "#The test score for Monte Carlo is -21023213.092998773 which shows that this model is overfitted\n",
    "#After ridge regression, the test scores are shown below. They are much better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Score for Linear Regression= 0.9423880620219615\n",
      "Test Score for Linear Regression= 0.823418320290348\n"
     ]
    }
   ],
   "source": [
    "#Splitting the data into training and testing and then printing out the scores\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split( X, y )\n",
    "lr.fit(X_train,y_train)\n",
    "print('Train Score for Linear Regression=',lr.score(X_train,y_train))\n",
    "print('Test Score for Linear Regression=',lr.score(X_test,y_test))\n",
    "\n",
    "#After splitting the model into training and testing data and performing linear regression, it can be seen the test score is 0.823\n",
    "#The test Ridge regression results are slightly lower(0.771,0.775,0.779) compared to the test regression score after splitting the data (0.823)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   rank_test_score  mean_test_score param_ridge__alpha\n",
      "0                3         0.781877                0.5\n",
      "1                2         0.782923                  1\n",
      "2                1         0.784711                  2\n"
     ]
    }
   ],
   "source": [
    "#4.\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "fullModel = Pipeline([\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"ridge\", Ridge())\n",
    "])\n",
    "\n",
    "# set up dictionary for grid search\n",
    "param_grid={'ridge__alpha':[0.5,1.,2.]}\n",
    "# set up cross-validation shuffles\n",
    "shuffle_split = ShuffleSplit(test_size=0.5, n_splits=100)\n",
    "# set up search\n",
    "grid_search=GridSearchCV(fullModel,param_grid,cv=shuffle_split,scoring=score_used, \n",
    "                              return_train_score=True,n_jobs=1)\n",
    "# implement search\n",
    "grid_search.fit(X,y)\n",
    "# move results into DataFrame\n",
    "results = pd.DataFrame(grid_search.cv_results_)\n",
    "print(results[['rank_test_score','mean_test_score','param_ridge__alpha']])\n",
    "\n",
    "#The test values after StandardScaler have improved slightly compared to the Ridge Regression model"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}