XGBoost Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from scipy import stats\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.read_csv('X_train_prep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.read_csv('X_test_prep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X= pd.read_csv('/Users/mi0310/Desktop/JPMC/train.csv')\n",
    "Y_train = X[\"Fare\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Mumbai       1447\n",
       "Delhi        1424\n",
       "Lucknow      1420\n",
       "Patna         872\n",
       "Chennai       809\n",
       "Kolkata       770\n",
       "Hyderabad     758\n",
       "Name: To, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[\"To\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "distance = pd.read_csv('/Users/mi0310/Downloads/dist (1).csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "city_map = {\"Delhi\": 1, \"Mumbai\": 2, \"Lucknow\": 3, \"Kolkata\": 4, \"Hyderabad\": 5, \"Patna\":6, \"Chennai\":7}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "distance[\"From\"] = distance[\"from\"]\n",
    "distance[\"To\"] = distance[\"to\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "del distance[\"from\"]\n",
    "del distance[\"to\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distance</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1446.50</td>\n",
       "      <td>Lucknow</td>\n",
       "      <td>Mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1419.33</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>Mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>710.00</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>Mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2054.77</td>\n",
       "      <td>Kolkata</td>\n",
       "      <td>Mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1744.17</td>\n",
       "      <td>Patna</td>\n",
       "      <td>Mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1334.00</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>Mumbai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>536.00</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>Lucknow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1434.00</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>Lucknow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>991.00</td>\n",
       "      <td>Kolkata</td>\n",
       "      <td>Lucknow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>539.00</td>\n",
       "      <td>Patna</td>\n",
       "      <td>Lucknow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2057.30</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>Lucknow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1568.00</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>Delhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1472.00</td>\n",
       "      <td>Kolkata</td>\n",
       "      <td>Delhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1073.00</td>\n",
       "      <td>Patna</td>\n",
       "      <td>Delhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2191.00</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>Delhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1491.67</td>\n",
       "      <td>Kolkata</td>\n",
       "      <td>Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1507.00</td>\n",
       "      <td>Patna</td>\n",
       "      <td>Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>627.00</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>580.00</td>\n",
       "      <td>Kolkata</td>\n",
       "      <td>Patna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2042.00</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>Patna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1671.00</td>\n",
       "      <td>Kolkata</td>\n",
       "      <td>Chennai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1446.50</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Lucknow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1419.33</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Delhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>710.00</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2054.77</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Kolkata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1744.17</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Patna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1334.00</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Chennai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>536.00</td>\n",
       "      <td>Lucknow</td>\n",
       "      <td>Delhi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1434.00</td>\n",
       "      <td>Lucknow</td>\n",
       "      <td>Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>991.00</td>\n",
       "      <td>Lucknow</td>\n",
       "      <td>Kolkata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>539.00</td>\n",
       "      <td>Lucknow</td>\n",
       "      <td>Patna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2057.30</td>\n",
       "      <td>Lucknow</td>\n",
       "      <td>Chennai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1568.00</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1472.00</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>Kolkata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1073.00</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>Patna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2191.00</td>\n",
       "      <td>Delhi</td>\n",
       "      <td>Chennai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1491.67</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>Kolkata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1507.00</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>Patna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>627.00</td>\n",
       "      <td>Hyderabad</td>\n",
       "      <td>Chennai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>580.00</td>\n",
       "      <td>Patna</td>\n",
       "      <td>Kolkata</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2042.00</td>\n",
       "      <td>Patna</td>\n",
       "      <td>Chennai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1671.00</td>\n",
       "      <td>Chennai</td>\n",
       "      <td>Kolkata</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    distance       From         To\n",
       "0    1446.50    Lucknow     Mumbai\n",
       "1    1419.33      Delhi     Mumbai\n",
       "2     710.00  Hyderabad     Mumbai\n",
       "3    2054.77    Kolkata     Mumbai\n",
       "4    1744.17      Patna     Mumbai\n",
       "5    1334.00    Chennai     Mumbai\n",
       "6     536.00      Delhi    Lucknow\n",
       "7    1434.00  Hyderabad    Lucknow\n",
       "8     991.00    Kolkata    Lucknow\n",
       "9     539.00      Patna    Lucknow\n",
       "10   2057.30    Chennai    Lucknow\n",
       "11   1568.00  Hyderabad      Delhi\n",
       "12   1472.00    Kolkata      Delhi\n",
       "13   1073.00      Patna      Delhi\n",
       "14   2191.00    Chennai      Delhi\n",
       "15   1491.67    Kolkata  Hyderabad\n",
       "16   1507.00      Patna  Hyderabad\n",
       "17    627.00    Chennai  Hyderabad\n",
       "18    580.00    Kolkata      Patna\n",
       "19   2042.00    Chennai      Patna\n",
       "20   1671.00    Kolkata    Chennai\n",
       "21   1446.50     Mumbai    Lucknow\n",
       "22   1419.33     Mumbai      Delhi\n",
       "23    710.00     Mumbai  Hyderabad\n",
       "24   2054.77     Mumbai    Kolkata\n",
       "25   1744.17     Mumbai      Patna\n",
       "26   1334.00     Mumbai    Chennai\n",
       "27    536.00    Lucknow      Delhi\n",
       "28   1434.00    Lucknow  Hyderabad\n",
       "29    991.00    Lucknow    Kolkata\n",
       "30    539.00    Lucknow      Patna\n",
       "31   2057.30    Lucknow    Chennai\n",
       "32   1568.00      Delhi  Hyderabad\n",
       "33   1472.00      Delhi    Kolkata\n",
       "34   1073.00      Delhi      Patna\n",
       "35   2191.00      Delhi    Chennai\n",
       "36   1491.67  Hyderabad    Kolkata\n",
       "37   1507.00  Hyderabad      Patna\n",
       "38    627.00  Hyderabad    Chennai\n",
       "39    580.00      Patna    Kolkata\n",
       "40   2042.00      Patna    Chennai\n",
       "41   1671.00    Chennai    Kolkata"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "distance['From'] = distance['From'].map(city_map)\n",
    "distance['To'] = distance['To'].map(city_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distance</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1446.50</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1419.33</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>710.00</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2054.77</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1744.17</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1334.00</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>536.00</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1434.00</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>991.00</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>539.00</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2057.30</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1568.00</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1472.00</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1073.00</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2191.00</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1491.67</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1507.00</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>627.00</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>580.00</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2042.00</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1671.00</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1446.50</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1419.33</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>710.00</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2054.77</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1744.17</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1334.00</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>536.00</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1434.00</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>991.00</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>539.00</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>2057.30</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1568.00</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1472.00</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1073.00</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2191.00</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1491.67</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1507.00</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>627.00</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>580.00</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>2042.00</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1671.00</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    distance  From  To\n",
       "0    1446.50     3   2\n",
       "1    1419.33     1   2\n",
       "2     710.00     5   2\n",
       "3    2054.77     4   2\n",
       "4    1744.17     6   2\n",
       "5    1334.00     7   2\n",
       "6     536.00     1   3\n",
       "7    1434.00     5   3\n",
       "8     991.00     4   3\n",
       "9     539.00     6   3\n",
       "10   2057.30     7   3\n",
       "11   1568.00     5   1\n",
       "12   1472.00     4   1\n",
       "13   1073.00     6   1\n",
       "14   2191.00     7   1\n",
       "15   1491.67     4   5\n",
       "16   1507.00     6   5\n",
       "17    627.00     7   5\n",
       "18    580.00     4   6\n",
       "19   2042.00     7   6\n",
       "20   1671.00     4   7\n",
       "21   1446.50     2   3\n",
       "22   1419.33     2   1\n",
       "23    710.00     2   5\n",
       "24   2054.77     2   4\n",
       "25   1744.17     2   6\n",
       "26   1334.00     2   7\n",
       "27    536.00     3   1\n",
       "28   1434.00     3   5\n",
       "29    991.00     3   4\n",
       "30    539.00     3   6\n",
       "31   2057.30     3   7\n",
       "32   1568.00     1   5\n",
       "33   1472.00     1   4\n",
       "34   1073.00     1   6\n",
       "35   2191.00     1   7\n",
       "36   1491.67     5   4\n",
       "37   1507.00     5   6\n",
       "38    627.00     5   7\n",
       "39    580.00     6   4\n",
       "40   2042.00     6   7\n",
       "41   1671.00     7   4"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "s1 = pd.merge(X_train, distance, how='left', on=['From', 'To'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "s2 = pd.merge(X_test, distance, how='left', on=['From', 'To'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "From                        int64\n",
       "To                          int64\n",
       "Class                       int64\n",
       "Age_at_Booking              int64\n",
       "Booking_to_flight           int64\n",
       "Booking Year                int64\n",
       "Booking Month               int64\n",
       "Booking Dayofweek           int64\n",
       "Booking Is_month_end         bool\n",
       "Booking Is_month_start       bool\n",
       "Flight Month                int64\n",
       "Flight Dayofweek            int64\n",
       "Flight Is_month_end          bool\n",
       "Flight Is_month_start        bool\n",
       "Time Label                  int64\n",
       "distance                  float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = s1\n",
    "X_test = s2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_train[\"Flight Time\"]\n",
    "del X_train[\"Flight Year\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_transform = ['Class', 'From', 'To', 'Time Label']\n",
    "X = pd.get_dummies(X, columns=cols_to_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_transform = ['Class', 'From', 'To', 'Time Label']\n",
    "X_test = pd.get_dummies(X_test, columns=cols_to_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "regr = RandomForestRegressor(max_depth=25, random_state=0,n_estimators=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=25,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=1,\n",
       "           oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = regr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1736.15246,  2116.23454,  7665.24192, ...,  7981.8579 ,\n",
       "        7408.52574, 11156.16658])"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(\"y_pred_randomforest_distancesquared.csv\", Y_pred, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_csv('X_preprocessed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test.to_csv('X_test_preprocessed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[\"squaredistance\"] = pow(X_train[\"distance\"],2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test[\"squaredistance\"] = pow(X_test[\"distance\"],2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_wd = X_train[\"distance\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_wd = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_wd[\"distance\"] = X_train[\"distance\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=25,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=1,\n",
       "           oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr.fit(X, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = regr.predict(X_test[[\"distance\",\"squaredistance\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(\"y_pred_randomforest_distancesquared_less.csv\", Y_pred, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[\"Fare\"] = Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "s1[\"Fare\"] = Y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "337\n",
      "334\n",
      "168\n",
      "182\n",
      "196\n",
      "183\n",
      "348\n",
      "344\n",
      "174\n",
      "160\n",
      "181\n",
      "166\n",
      "357\n",
      "335\n",
      "147\n",
      "154\n",
      "173\n",
      "181\n",
      "158\n",
      "207\n",
      "161\n",
      "84\n",
      "103\n",
      "81\n",
      "192\n",
      "158\n",
      "171\n",
      "89\n",
      "93\n",
      "87\n",
      "163\n",
      "185\n",
      "191\n",
      "68\n",
      "76\n",
      "81\n",
      "158\n",
      "175\n",
      "171\n",
      "94\n",
      "72\n",
      "94\n"
     ]
    }
   ],
   "source": [
    "X = s1[s1[\"From\"]==8]\n",
    "for i in range(1,8):\n",
    "    for j in range(1,8):\n",
    "        X2 = s1[s1[\"From\"]==i]\n",
    "        X2 = X2[X2[\"To\"]==j]\n",
    "        if(len(X2)>0):\n",
    "            low = np.percentile(X2[\"Fare\"],1.5)\n",
    "            high = np.percentile(X2[\"Fare\"],98.5)\n",
    "            X2 = X2[X2[\"Fare\"]>=low]\n",
    "            X2 = X2[X2[\"Fare\"]<=high]\n",
    "            print(len(X2))\n",
    "        X=X.append(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age_at_Booking              int64\n",
       "Booking_to_flight           int64\n",
       "Booking Year                int64\n",
       "Booking Month               int64\n",
       "Booking Dayofweek           int64\n",
       "Booking Is_month_end         bool\n",
       "Booking Is_month_start       bool\n",
       "Flight Month                int64\n",
       "Flight Dayofweek            int64\n",
       "Flight Is_month_end          bool\n",
       "Flight Is_month_start        bool\n",
       "distance                  float64\n",
       "Fare                      float64\n",
       "Class_0                     uint8\n",
       "Class_1                     uint8\n",
       "From_1                      uint8\n",
       "From_2                      uint8\n",
       "From_3                      uint8\n",
       "From_4                      uint8\n",
       "From_5                      uint8\n",
       "From_6                      uint8\n",
       "From_7                      uint8\n",
       "To_1                        uint8\n",
       "To_2                        uint8\n",
       "To_3                        uint8\n",
       "To_4                        uint8\n",
       "To_5                        uint8\n",
       "To_6                        uint8\n",
       "To_7                        uint8\n",
       "Time Label_0                uint8\n",
       "Time Label_1                uint8\n",
       "Time Label_2                uint8\n",
       "Time Label_3                uint8\n",
       "Time Label_4                uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = X[\"Fare\"]\n",
    "del X[\"Fare\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_test[\"squaredistance\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X[\"Fare\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=25,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=1,\n",
       "           oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr.fit(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = regr.predict(X_test)\n",
    "np.savetxt(\"y_pred_randomforest_distancesquared_less_outliersno.csv\", Y_pred, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7500"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = [{'n_estimators': [75, 100],'max_features':[10, 20],'max_depth':[20, 30]}]\n",
    "forest_reg = RandomForestRegressor(n_jobs=-1)\n",
    "grid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring='neg_mean_squared_error')\n",
    "grid_search.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "regr = RandomForestRegressor(max_depth=20, max_features=20,  min_samples_leaf=1,random_state=0,n_estimators=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,\n",
       "           max_features=20, max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=100, n_jobs=-1, oob_score=False, random_state=None,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=20,\n",
       "           max_features=20, max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=100, n_jobs=1, oob_score=False, random_state=0,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = regr.predict(X_test)\n",
    "np.savetxt(\"y_pred_randomforest_distancesquared_less_outliersno_gridsearch.csv\", Y_pred, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age_at_Booking              int64\n",
       "Booking_to_flight           int64\n",
       "Booking Year                int64\n",
       "Booking Month               int64\n",
       "Booking Dayofweek           int64\n",
       "Booking Is_month_end         bool\n",
       "Booking Is_month_start       bool\n",
       "Flight Month                int64\n",
       "Flight Dayofweek            int64\n",
       "Flight Is_month_end          bool\n",
       "Flight Is_month_start        bool\n",
       "distance                  float64\n",
       "Class_0                     uint8\n",
       "Class_1                     uint8\n",
       "From_1                      uint8\n",
       "From_2                      uint8\n",
       "From_3                      uint8\n",
       "From_4                      uint8\n",
       "From_5                      uint8\n",
       "From_6                      uint8\n",
       "From_7                      uint8\n",
       "To_1                        uint8\n",
       "To_2                        uint8\n",
       "To_3                        uint8\n",
       "To_4                        uint8\n",
       "To_5                        uint8\n",
       "To_6                        uint8\n",
       "To_7                        uint8\n",
       "Time Label_0                uint8\n",
       "Time Label_1                uint8\n",
       "Time Label_2                uint8\n",
       "Time Label_3                uint8\n",
       "Time Label_4                uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_train[\"Fare\"]\n",
    "del X_train[\"squaredistance\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.read_csv('X_preprcessed_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.read_csv('X_train_preprcessed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "del X_train[\"Unnamed: 0\"]\n",
    "del X_test[\"Unnamed: 0\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'plot_importance' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-171-2200abb95def>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplot_importance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mregr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mpyplot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'plot_importance' is not defined"
     ]
    }
   ],
   "source": [
    "plot_importance(regr)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "\n",
    "xgbc = XGBClassifier(**xgb_params, verbose_eval=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbc.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age_at_Booking            int64\n",
       "Booking_to_flight         int64\n",
       "Booking Year              int64\n",
       "Booking Month             int64\n",
       "Booking Dayofweek         int64\n",
       "Booking Is_month_end      int64\n",
       "Booking Is_month_start    int64\n",
       "Flight Month              int64\n",
       "Flight Dayofweek          int64\n",
       "Flight Is_month_end       int64\n",
       "Flight Is_month_start     int64\n",
       "distance                  int64\n",
       "Class_0                   int64\n",
       "Class_1                   int64\n",
       "From_1                    int64\n",
       "From_2                    int64\n",
       "From_3                    int64\n",
       "From_4                    int64\n",
       "From_5                    int64\n",
       "From_6                    int64\n",
       "From_7                    int64\n",
       "To_1                      int64\n",
       "To_2                      int64\n",
       "To_3                      int64\n",
       "To_4                      int64\n",
       "To_5                      int64\n",
       "To_6                      int64\n",
       "To_7                      int64\n",
       "Time Label_0              int64\n",
       "Time Label_1              int64\n",
       "Time Label_2              int64\n",
       "Time Label_3              int64\n",
       "Time Label_4              int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tr.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7500"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'max_depth': 9, \n",
    "    'n_estimators' : 500,\n",
    "    'learning_rate': 0.04,\n",
    "    'min_child_weight' : 2,\n",
    "    'objective':'reg:linear',\n",
    "     'colsample_bylevel':0.8,\n",
    "     'colsample_bytree':0.8,\n",
    "    'silent': False,\n",
    "}\n",
    "import xgboost as xgb\n",
    "dtrain = xgb.DMatrix(X_train, Y_train)\n",
    "dtest=xgb.DMatrix(X_test)\n",
    "model=xgb.train(xgb_params,dtrain, num_boost_round=178)\n",
    "preds = model.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(\"Y_preddd_xgboosterrr.csv\", preds, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "\n",
    "params = {'min_child_weight':[4,5], 'gamma':[i/10.0 for i in range(3,6)],  'subsample':[i/10.0 for i in range(6,11)],\n",
    "'colsample_bytree':[i/10.0 for i in range(6,11)], 'max_depth': [2,3,4],'n_estimators' : [100,200,300,400]}\n",
    "\n",
    "xgbc = xgb.XGBRegressor(nthread=-1) \n",
    "\n",
    "grid = GridSearchCV(xgbc, params)\n",
    "grid.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pred1 = pd.read_csv('Y_preddd_xgboost.csv')\n",
    "pred2 = pd.read_csv('y_pred_randomforest_distancesquared_less_outliersno_gridsearch.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.savetxt(\"xgboost+randomforest.csv\", pred, delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred1 = pred1.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pred1+pred2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3651.401401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4154.380712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14870.034862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9221.935714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16812.005388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>17639.838506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20820.916803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10086.338931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>10536.819172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>6279.574559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>18309.422371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>8771.621359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>8524.528553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>17716.567796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>7705.253492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>13707.800134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>14345.600648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18416.422414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12531.418587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>34906.793428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5377.837183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>16885.176921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>26851.458355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>9640.785399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>19666.897828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>3347.741714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>10665.943784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>9542.002985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>25276.353843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>19959.493404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2470</th>\n",
       "      <td>18226.858346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2471</th>\n",
       "      <td>9693.530070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2472</th>\n",
       "      <td>9682.296497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2473</th>\n",
       "      <td>17031.864409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2474</th>\n",
       "      <td>10629.187557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2475</th>\n",
       "      <td>10353.570541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2476</th>\n",
       "      <td>3784.186965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2477</th>\n",
       "      <td>22563.002605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2478</th>\n",
       "      <td>16243.591923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2479</th>\n",
       "      <td>10737.691733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2480</th>\n",
       "      <td>15378.469751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2481</th>\n",
       "      <td>12905.610725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2482</th>\n",
       "      <td>21437.604314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2483</th>\n",
       "      <td>39892.877637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2484</th>\n",
       "      <td>3712.671532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>10141.444850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2486</th>\n",
       "      <td>47225.632397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2487</th>\n",
       "      <td>10210.207132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2488</th>\n",
       "      <td>5703.693967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2489</th>\n",
       "      <td>24446.806961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2490</th>\n",
       "      <td>10788.145699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2491</th>\n",
       "      <td>46885.607606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2492</th>\n",
       "      <td>19338.783570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2493</th>\n",
       "      <td>15456.072972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2494</th>\n",
       "      <td>16752.487764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2495</th>\n",
       "      <td>13188.926174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2496</th>\n",
       "      <td>49806.168878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2497</th>\n",
       "      <td>15795.169537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2498</th>\n",
       "      <td>13849.493505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2499</th>\n",
       "      <td>21428.401411</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2500 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "0      3651.401401\n",
       "1      4154.380712\n",
       "2     14870.034862\n",
       "3      9221.935714\n",
       "4     16812.005388\n",
       "5     17639.838506\n",
       "6     20820.916803\n",
       "7     10086.338931\n",
       "8     10536.819172\n",
       "9      6279.574559\n",
       "10    18309.422371\n",
       "11     8771.621359\n",
       "12     8524.528553\n",
       "13    17716.567796\n",
       "14     7705.253492\n",
       "15    13707.800134\n",
       "16    14345.600648\n",
       "17    18416.422414\n",
       "18    12531.418587\n",
       "19    34906.793428\n",
       "20     5377.837183\n",
       "21    16885.176921\n",
       "22    26851.458355\n",
       "23     9640.785399\n",
       "24    19666.897828\n",
       "25     3347.741714\n",
       "26    10665.943784\n",
       "27     9542.002985\n",
       "28    25276.353843\n",
       "29    19959.493404\n",
       "...            ...\n",
       "2470  18226.858346\n",
       "2471   9693.530070\n",
       "2472   9682.296497\n",
       "2473  17031.864409\n",
       "2474  10629.187557\n",
       "2475  10353.570541\n",
       "2476   3784.186965\n",
       "2477  22563.002605\n",
       "2478  16243.591923\n",
       "2479  10737.691733\n",
       "2480  15378.469751\n",
       "2481  12905.610725\n",
       "2482  21437.604314\n",
       "2483  39892.877637\n",
       "2484   3712.671532\n",
       "2485  10141.444850\n",
       "2486  47225.632397\n",
       "2487  10210.207132\n",
       "2488   5703.693967\n",
       "2489  24446.806961\n",
       "2490  10788.145699\n",
       "2491  46885.607606\n",
       "2492  19338.783570\n",
       "2493  15456.072972\n",
       "2494  16752.487764\n",
       "2495  13188.926174\n",
       "2496  49806.168878\n",
       "2497  15795.169537\n",
       "2498  13849.493505\n",
       "2499  21428.401411\n",
       "\n",
       "[2500 rows x 1 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Booster' object has no attribute 'feature_importances_'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-6c8de3470678>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mxgboost\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mXGBClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mmatplotlib\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpyplot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mpyplot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mpyplot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Booster' object has no attribute 'feature_importances_'"
     ]
    }
   ],
   "source": [
    "from numpy import loadtxt\n",
    "from xgboost import XGBClassifier\n",
    "from matplotlib import pyplot\n",
    "pyplot.bar(range(len(model.feature_importances_)), model.feature_importances_)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Booster' object has no attribute 'feature_importances_'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-46d89b1c7830>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'Booster' object has no attribute 'feature_importances_'"
     ]
    }
   ],
   "source": [
    "print(model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Age_at_Booking              int64\n",
       "Booking_to_flight           int64\n",
       "Booking Year                int64\n",
       "Booking Month               int64\n",
       "Booking Dayofweek           int64\n",
       "Booking Is_month_end         bool\n",
       "Booking Is_month_start       bool\n",
       "Flight Month                int64\n",
       "Flight Dayofweek            int64\n",
       "Flight Is_month_end          bool\n",
       "Flight Is_month_start        bool\n",
       "distance                  float64\n",
       "Class_0                     int64\n",
       "Class_1                     int64\n",
       "From_1                      int64\n",
       "From_2                      int64\n",
       "From_3                      int64\n",
       "From_4                      int64\n",
       "From_5                      int64\n",
       "From_6                      int64\n",
       "From_7                      int64\n",
       "To_1                        int64\n",
       "To_2                        int64\n",
       "To_3                        int64\n",
       "To_4                        int64\n",
       "To_5                        int64\n",
       "To_6                        int64\n",
       "To_7                        int64\n",
       "Time Label_0                int64\n",
       "Time Label_1                int64\n",
       "Time Label_2                int64\n",
       "Time Label_3                int64\n",
       "Time Label_4                int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}