1_laptop.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting up to speed with Dask\n",
    "\n",
    "## Part 1: Not Dask\n",
    "\n",
    "Let's do our usual analysis on a laptop-sized machine with a dataset that fits comfortably in memory.\n",
    "\n",
    "AWS EC2 instance type: r5.xlarge (2 CPU, 16GB RAM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import s3fs\n",
    "from pathlib import Path\n",
    "\n",
    "# change this path if you changed in Part 0\n",
    "data_path = Path('data')\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and explore data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxi_dtypes = {\n",
    "    'store_and_fwd_flag': str,\n",
    "    'RatecodeID': 'float64',\n",
    "    'VendorID': 'float64',\n",
    "    'passenger_count': 'float64',\n",
    "    'payment_type': 'float64',\n",
    "}\n",
    "\n",
    "def load_csv(f):\n",
    "    \"\"\" Your good ol' pd.read_csv \"\"\"\n",
    "    return pd.read_csv(\n",
    "        f,\n",
    "        dtype=taxi_dtypes, \n",
    "        parse_dates=['tpep_pickup_datetime', 'tpep_dropoff_datetime'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.2 s, sys: 1.34 s, total: 13.5 s\n",
      "Wall time: 14.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "taxi = load_csv(data_path/'yellow_tripdata_2019-01.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7667792, 1.548894112)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rows and appx. memory size in GB\n",
    "len(taxi), taxi.memory_usage(deep=True).sum() / 1e9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>fare_amount</th>\n",
       "      <th>extra</th>\n",
       "      <th>mta_tax</th>\n",
       "      <th>tip_amount</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-01-01 00:46:40</td>\n",
       "      <td>2019-01-01 00:53:20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>151</td>\n",
       "      <td>239</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.65</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>9.95</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-01-01 00:59:47</td>\n",
       "      <td>2019-01-01 01:18:59</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>239</td>\n",
       "      <td>246</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>16.30</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2018-12-21 13:48:30</td>\n",
       "      <td>2018-12-21 13:52:40</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>236</td>\n",
       "      <td>236</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>5.80</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2018-11-28 15:52:25</td>\n",
       "      <td>2018-11-28 15:55:45</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>N</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>7.55</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2018-11-28 15:56:57</td>\n",
       "      <td>2018-11-28 15:58:33</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>N</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>2.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>55.55</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0       1.0  2019-01-01 00:46:40   2019-01-01 00:53:20              1.0   \n",
       "1       1.0  2019-01-01 00:59:47   2019-01-01 01:18:59              1.0   \n",
       "2       2.0  2018-12-21 13:48:30   2018-12-21 13:52:40              3.0   \n",
       "3       2.0  2018-11-28 15:52:25   2018-11-28 15:55:45              5.0   \n",
       "4       2.0  2018-11-28 15:56:57   2018-11-28 15:58:33              5.0   \n",
       "\n",
       "   trip_distance  RatecodeID store_and_fwd_flag  PULocationID  DOLocationID  \\\n",
       "0            1.5         1.0                  N           151           239   \n",
       "1            2.6         1.0                  N           239           246   \n",
       "2            0.0         1.0                  N           236           236   \n",
       "3            0.0         1.0                  N           193           193   \n",
       "4            0.0         2.0                  N           193           193   \n",
       "\n",
       "   payment_type  fare_amount  extra  mta_tax  tip_amount  tolls_amount  \\\n",
       "0           1.0          7.0    0.5      0.5        1.65           0.0   \n",
       "1           1.0         14.0    0.5      0.5        1.00           0.0   \n",
       "2           1.0          4.5    0.5      0.5        0.00           0.0   \n",
       "3           2.0          3.5    0.5      0.5        0.00           0.0   \n",
       "4           2.0         52.0    0.0      0.5        0.00           0.0   \n",
       "\n",
       "   improvement_surcharge  total_amount  congestion_surcharge  \n",
       "0                    0.3          9.95                   NaN  \n",
       "1                    0.3         16.30                   NaN  \n",
       "2                    0.3          5.80                   NaN  \n",
       "3                    0.3          7.55                   NaN  \n",
       "4                    0.3         55.55                   NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxi.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VendorID                        float64\n",
       "tpep_pickup_datetime     datetime64[ns]\n",
       "tpep_dropoff_datetime    datetime64[ns]\n",
       "passenger_count                 float64\n",
       "trip_distance                   float64\n",
       "RatecodeID                      float64\n",
       "store_and_fwd_flag               object\n",
       "PULocationID                      int64\n",
       "DOLocationID                      int64\n",
       "payment_type                    float64\n",
       "fare_amount                     float64\n",
       "extra                           float64\n",
       "mta_tax                         float64\n",
       "tip_amount                      float64\n",
       "tolls_amount                    float64\n",
       "improvement_surcharge           float64\n",
       "total_amount                    float64\n",
       "congestion_surcharge            float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxi.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.57 s, sys: 207 ms, total: 4.78 s\n",
      "Wall time: 4.78 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>VendorID</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>1.637</td>\n",
       "      <td>0.540</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>passenger_count</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>1.567</td>\n",
       "      <td>1.224</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>9.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trip_distance</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>2.801</td>\n",
       "      <td>3.738</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.90</td>\n",
       "      <td>1.53</td>\n",
       "      <td>2.80</td>\n",
       "      <td>831.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RatecodeID</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>1.058</td>\n",
       "      <td>0.678</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>99.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PULocationID</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>165.501</td>\n",
       "      <td>66.392</td>\n",
       "      <td>1.0</td>\n",
       "      <td>130.00</td>\n",
       "      <td>162.00</td>\n",
       "      <td>234.00</td>\n",
       "      <td>265.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DOLocationID</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>163.753</td>\n",
       "      <td>70.364</td>\n",
       "      <td>1.0</td>\n",
       "      <td>113.00</td>\n",
       "      <td>162.00</td>\n",
       "      <td>234.00</td>\n",
       "      <td>265.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>payment_type</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>1.292</td>\n",
       "      <td>0.473</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2.00</td>\n",
       "      <td>4.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fare_amount</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>12.409</td>\n",
       "      <td>262.072</td>\n",
       "      <td>-362.0</td>\n",
       "      <td>6.00</td>\n",
       "      <td>8.50</td>\n",
       "      <td>13.50</td>\n",
       "      <td>623259.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>extra</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>0.328</td>\n",
       "      <td>0.507</td>\n",
       "      <td>-60.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.50</td>\n",
       "      <td>535.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mta_tax</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>0.497</td>\n",
       "      <td>0.053</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.50</td>\n",
       "      <td>60.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tip_amount</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>1.827</td>\n",
       "      <td>2.501</td>\n",
       "      <td>-63.5</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.43</td>\n",
       "      <td>2.33</td>\n",
       "      <td>787.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tolls_amount</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>0.317</td>\n",
       "      <td>2.024</td>\n",
       "      <td>-70.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3288.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>0.299</td>\n",
       "      <td>0.019</td>\n",
       "      <td>-0.3</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_amount</th>\n",
       "      <td>7667792.0</td>\n",
       "      <td>15.682</td>\n",
       "      <td>262.293</td>\n",
       "      <td>-362.8</td>\n",
       "      <td>8.19</td>\n",
       "      <td>11.27</td>\n",
       "      <td>16.56</td>\n",
       "      <td>623261.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <td>2811814.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           count     mean      std    min     25%     50%  \\\n",
       "VendorID               7667792.0    1.637    0.540    1.0    1.00    2.00   \n",
       "passenger_count        7667792.0    1.567    1.224    0.0    1.00    1.00   \n",
       "trip_distance          7667792.0    2.801    3.738    0.0    0.90    1.53   \n",
       "RatecodeID             7667792.0    1.058    0.678    1.0    1.00    1.00   \n",
       "PULocationID           7667792.0  165.501   66.392    1.0  130.00  162.00   \n",
       "DOLocationID           7667792.0  163.753   70.364    1.0  113.00  162.00   \n",
       "payment_type           7667792.0    1.292    0.473    1.0    1.00    1.00   \n",
       "fare_amount            7667792.0   12.409  262.072 -362.0    6.00    8.50   \n",
       "extra                  7667792.0    0.328    0.507  -60.0    0.00    0.00   \n",
       "mta_tax                7667792.0    0.497    0.053   -0.5    0.50    0.50   \n",
       "tip_amount             7667792.0    1.827    2.501  -63.5    0.00    1.43   \n",
       "tolls_amount           7667792.0    0.317    2.024  -70.0    0.00    0.00   \n",
       "improvement_surcharge  7667792.0    0.299    0.019   -0.3    0.30    0.30   \n",
       "total_amount           7667792.0   15.682  262.293 -362.8    8.19   11.27   \n",
       "congestion_surcharge   2811814.0    0.000    0.009    0.0    0.00    0.00   \n",
       "\n",
       "                          75%        max  \n",
       "VendorID                 2.00       4.00  \n",
       "passenger_count          2.00       9.00  \n",
       "trip_distance            2.80     831.80  \n",
       "RatecodeID               1.00      99.00  \n",
       "PULocationID           234.00     265.00  \n",
       "DOLocationID           234.00     265.00  \n",
       "payment_type             2.00       4.00  \n",
       "fare_amount             13.50  623259.86  \n",
       "extra                    0.50     535.38  \n",
       "mta_tax                  0.50      60.80  \n",
       "tip_amount               2.33     787.25  \n",
       "tolls_amount             0.00    3288.00  \n",
       "improvement_surcharge    0.30       0.60  \n",
       "total_amount            16.56  623261.66  \n",
       "congestion_surcharge     0.00       2.50  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time \n",
    "np.round(taxi.describe(), 3).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature engineering\n",
    "\n",
    "Adapted from this [Kaggle kernel](https://www.kaggle.com/gaborfodor/from-eda-to-the-top-lb-0-367#Feature-Extraction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_features(df):\n",
    "    \"\"\" Create some features \"\"\"\n",
    "    df['pickup_weekday'] = df.tpep_pickup_datetime.dt.weekday\n",
    "    df['pickup_weekofyear'] = df.tpep_pickup_datetime.dt.weekofyear\n",
    "    df['pickup_hour'] = df.tpep_pickup_datetime.dt.hour\n",
    "    df['pickup_minute'] = df.tpep_pickup_datetime.dt.minute\n",
    "    df['pickup_year_seconds'] = (df.tpep_pickup_datetime - datetime.datetime(2019, 1, 1, 0, 0, 0)).dt.seconds\n",
    "    df['pickup_week_hour'] = (df.pickup_weekday * 24) + df.pickup_hour\n",
    "    df['store_and_fwd_flag'] = (df.store_and_fwd_flag == 'Y').astype(int)\n",
    "    df['VendorID'] = df.VendorID.fillna(-1)\n",
    "    df['RatecodeID'] = df.RatecodeID.fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.31 s, sys: 132 ms, total: 3.44 s\n",
      "Wall time: 3.44 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "make_features(taxi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>VendorID</th>\n",
       "      <th>tpep_pickup_datetime</th>\n",
       "      <th>tpep_dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>trip_distance</th>\n",
       "      <th>RatecodeID</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>PULocationID</th>\n",
       "      <th>DOLocationID</th>\n",
       "      <th>payment_type</th>\n",
       "      <th>...</th>\n",
       "      <th>tolls_amount</th>\n",
       "      <th>improvement_surcharge</th>\n",
       "      <th>total_amount</th>\n",
       "      <th>congestion_surcharge</th>\n",
       "      <th>pickup_weekday</th>\n",
       "      <th>pickup_weekofyear</th>\n",
       "      <th>pickup_hour</th>\n",
       "      <th>pickup_minute</th>\n",
       "      <th>pickup_year_seconds</th>\n",
       "      <th>pickup_week_hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-01-01 00:46:40</td>\n",
       "      <td>2019-01-01 00:53:20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>151</td>\n",
       "      <td>239</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>9.95</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>2800</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-01-01 00:59:47</td>\n",
       "      <td>2019-01-01 01:18:59</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>239</td>\n",
       "      <td>246</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>16.30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>3587</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2018-12-21 13:48:30</td>\n",
       "      <td>2018-12-21 13:52:40</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>236</td>\n",
       "      <td>236</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>5.80</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>51</td>\n",
       "      <td>13</td>\n",
       "      <td>48</td>\n",
       "      <td>49710</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2018-11-28 15:52:25</td>\n",
       "      <td>2018-11-28 15:55:45</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>7.55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>15</td>\n",
       "      <td>52</td>\n",
       "      <td>57145</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2018-11-28 15:56:57</td>\n",
       "      <td>2018-11-28 15:58:33</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>55.55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>15</td>\n",
       "      <td>56</td>\n",
       "      <td>57417</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   VendorID tpep_pickup_datetime tpep_dropoff_datetime  passenger_count  \\\n",
       "0       1.0  2019-01-01 00:46:40   2019-01-01 00:53:20              1.0   \n",
       "1       1.0  2019-01-01 00:59:47   2019-01-01 01:18:59              1.0   \n",
       "2       2.0  2018-12-21 13:48:30   2018-12-21 13:52:40              3.0   \n",
       "3       2.0  2018-11-28 15:52:25   2018-11-28 15:55:45              5.0   \n",
       "4       2.0  2018-11-28 15:56:57   2018-11-28 15:58:33              5.0   \n",
       "\n",
       "   trip_distance  RatecodeID  store_and_fwd_flag  PULocationID  DOLocationID  \\\n",
       "0            1.5         1.0                   0           151           239   \n",
       "1            2.6         1.0                   0           239           246   \n",
       "2            0.0         1.0                   0           236           236   \n",
       "3            0.0         1.0                   0           193           193   \n",
       "4            0.0         2.0                   0           193           193   \n",
       "\n",
       "   payment_type  ...  tolls_amount  improvement_surcharge  total_amount  \\\n",
       "0           1.0  ...           0.0                    0.3          9.95   \n",
       "1           1.0  ...           0.0                    0.3         16.30   \n",
       "2           1.0  ...           0.0                    0.3          5.80   \n",
       "3           2.0  ...           0.0                    0.3          7.55   \n",
       "4           2.0  ...           0.0                    0.3         55.55   \n",
       "\n",
       "   congestion_surcharge  pickup_weekday  pickup_weekofyear  pickup_hour  \\\n",
       "0                   NaN               1                  1            0   \n",
       "1                   NaN               1                  1            0   \n",
       "2                   NaN               4                 51           13   \n",
       "3                   NaN               2                 48           15   \n",
       "4                   NaN               2                 48           15   \n",
       "\n",
       "   pickup_minute  pickup_year_seconds  pickup_week_hour  \n",
       "0             46                 2800                24  \n",
       "1             59                 3587                24  \n",
       "2             48                49710               109  \n",
       "3             52                57145                63  \n",
       "4             56                57417                63  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxi.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# features and target column names\n",
    "numeric_feat = [\n",
    "    'pickup_weekday', \n",
    "    'pickup_weekofyear', \n",
    "    'pickup_hour', \n",
    "    'pickup_minute', \n",
    "    'pickup_year_seconds',\n",
    "    'pickup_week_hour', \n",
    "    'passenger_count',\n",
    "]\n",
    "categorical_feat = [\n",
    "    'VendorID', \n",
    "    'RatecodeID', \n",
    "    'store_and_fwd_flag',\n",
    "    'PULocationID',\n",
    "    'DOLocationID',\n",
    "]\n",
    "features = numeric_feat + categorical_feat\n",
    "y_col = 'total_amount'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from xgboost import XGBRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.09 s, sys: 517 ms, total: 2.61 s\n",
      "Wall time: 2.61 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    taxi[features], taxi[y_col], test_size=0.33, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBRegressor(\n",
    "    n_estimators=10, \n",
    "    max_depth=3, \n",
    "    learning_rate=0.1, \n",
    "    random_state=seed, \n",
    "    n_jobs=-1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[03:40:32] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1593723605503/work/src/gbm/gbtree.cc:139: Tree method is automatically selected to be 'approx' for faster speed. To use old behavior (exact greedy algorithm on single machine), set tree_method to 'exact'.\n",
      "CPU times: user 1min 18s, sys: 928 ms, total: 1min 19s\n",
      "Wall time: 1min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "_ = xgb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 956 ms, sys: 240 ms, total: 1.2 s\n",
      "Wall time: 1.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "29.79722106531764"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# get test RMSE\n",
    "preds = xgb.predict(X_test)\n",
    "np.sqrt(mean_squared_error(preds, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "<br>\n",
    "\n",
    "# Moar data!!\n",
    "\n",
    "![](https://i.chzbgr.com/full/6993318656/hC83012C2/analyze-all-the-data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('data/yellow_tripdata_2019-12.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-08.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-10.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-07.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-05.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-03.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-04.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-09.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-11.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-06.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-02.csv'),\n",
       " PosixPath('data/yellow_tripdata_2019-01.csv')]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(data_path.iterdir())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfs = []\n",
    "for f in data_path.iterdir():\n",
    "    df = load_csv(f)\n",
    "    dfs.append(df)\n",
    "taxi_2019 = pd.concat(dfs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://i.kym-cdn.com/entries/icons/original/000/010/437/Oneeternitylater.jpg)\n",
    "\n",
    "![](kernel_dead.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://memegenerator.net/img/instances/61402104.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dask to the rescue!\n",
    "\n",
    "See Part 2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}