2b_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import eviltransform\n",
    "import datetime\n",
    "\n",
    "# study area bounding box (bbox)\n",
    "YMIN, XMIN = eviltransform.wgs2gcj(34.234, 108.9415)\n",
    "YMAX, XMAX = eviltransform.wgs2gcj(34.241, 108.943)\n",
    "\n",
    "# latitude bounds for segments immediately above and below bounding box (between bbox and intersection)\n",
    "ABOVE = 34.241\n",
    "BELOW = 34.231\n",
    "\n",
    "# tighter latitude bounds for study road\n",
    "LLEFT = 108.9467 \n",
    "LRGHT = 108.9471\n",
    "\n",
    "# bounds for west feeder road\n",
    "WLEFT = 108.9393\n",
    "WNRTH = 34.2355\n",
    "WSOUT = 34.2345\n",
    "\n",
    "# bound for east feeder road\n",
    "ERGHT = 108.9486\n",
    "ENRTH = 34.236\n",
    "ESOUT = 34.237\n",
    "\n",
    "# corrections for bbox speed estimation\n",
    "NB_CORRECTION = 1.3\n",
    "SB_CORRECTION = -2.6\n",
    "\n",
    "# outlier threshold based on time needed/distance over study area and a sane speedlimit\n",
    "D_MAX = 3000 # meters\n",
    "T_MAX = pd.Timedelta(datetime.timedelta(minutes=10))\n",
    "V_MAX = 150 #km/h\n",
    "\n",
    "# grouper\n",
    "GPR = pd.Grouper(freq=\"5T\", key=\"timestamp1\", closed=\"left\", label=\"left\", base=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Identify data to process\n",
    "\n",
    "Given the October and November range of our datasets, \n",
    "\n",
    "We're asked to predict peak hour traffic December 1st, a Thursday, so we want to train our algorithm on comparable periods in comparable days. This leads us to exclude weekends and Fridays from our data. We also note that the first week of October is a week of national holidays in China, thus we exclude that week from our dataset as well. There appear to be no other holidays in this time-period so that leaves us with\n",
    "* 7 Thursdays\n",
    "* 8 Mondays, Wednesdays, and Thursdays each\n",
    "\n",
    "So these are the dates we wish to process.\n",
    "\n",
    "# Processing methodology\n",
    "\n",
    "1. \n",
    "1. Copy the dataset and remove the study area from the copy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "oct_mwf = ['10', '11', '12', '17', '18', '19', '24', '25', '26']\n",
    "oct_thu = ['13', '20', '27']\n",
    "nov_mwf = ['01', '02', '07', '08', '09', '14', '15', '16', '21', '22', '23', '28', '29', '30']\n",
    "nov_thu = ['03', '10', '17', '24']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'data/xian/gps_201610{}'.format(oct_thu[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 7.63 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def read_gps(filename, has_header=False):\n",
    "    # read and sort csv into order_id-then-chronological order\n",
    "    df = pd.read_csv(filename, names=None if has_header else ['driver_id', 'order_id', 'timestamp', 'longitude', 'latitude'])\n",
    "    df.drop(columns=['driver_id'], inplace=True)\n",
    "    df.sort_values(['order_id', 'timestamp'], inplace=True)\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    return df\n",
    "\n",
    "# dfy = read_gps(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get col name for lonlat pair\n",
    "def which_lon_lat(which=1):\n",
    "    lon = 'longitude{}'.format(which)\n",
    "    lat = 'latitude{}'.format(which)\n",
    "    return (lon, lat)   \n",
    "\n",
    "# study area (bounding box) filter to find true values y\n",
    "def bb_filter(df, which=1):\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] >= XMIN) & \\\n",
    "           (df[lon] <= XMAX) & \\\n",
    "           (df[lat] >= YMIN) & \\\n",
    "           (df[lat] <= YMAX)\n",
    "\n",
    "# simulate given data by deleting all gps coords inside bbox\n",
    "def delete_bbox(df):\n",
    "    return df.loc[\n",
    "        ~bb_filter(df, which='')\n",
    "    ,:].copy()\n",
    "\n",
    "# df = delete_bbox(dfy)\n",
    "# len(df), len(dfy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# am period\n",
    "def am_filter(df, timestamp_col, after=False): # include 5 min before study time, include 5 minutes after period if requested\n",
    "    return (df[timestamp_col] >= df[timestamp_col][0].replace(hour=5,minute=55,second=0)) & \\\n",
    "           ((df[timestamp_col] < df[timestamp_col][0].replace(hour=11,minute=5,second=0)) if (after) \\\n",
    "            else (df[timestamp_col] < df[timestamp_col][0].replace(hour=11,minute=0,second=0)))\n",
    "\n",
    "# pm period\n",
    "def pm_filter(df, timestamp_col, after=False): # include 5 min before\n",
    "    return (df[timestamp_col] >= df[timestamp_col][0].replace(hour=15,minute=55,second=0)) & \\\n",
    "           ((df[timestamp_col] < df[timestamp_col][0].replace(hour=21,minute=5,second=0)) if (after) \\\n",
    "            else (df[timestamp_col] < df[timestamp_col][0].replace(hour=21,minute=0,second=0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 7.39 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def make_deltas_split_ampm(df, drop_outliers=True, after=False):\n",
    "    # convert into gps steps\n",
    "    df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s').dt.tz_localize('UTC').dt.tz_convert('Asia/Shanghai')\n",
    "    df = pd.concat([df.rename(lambda x: x+'1', axis = 1),df.shift(-1).rename(lambda x: x+'2', axis = 1)],axis=1)\n",
    "    df.drop(df[df['order_id1'] != df['order_id2']].index, inplace=True)\n",
    "    df.drop(['order_id1', 'order_id2'], axis=1, inplace=True) # don't need these anymore\n",
    "    if drop_outliers:\n",
    "        df.drop(df.index[df['timestamp2'] - df['timestamp1'] > T_MAX], inplace=True)\n",
    "    dfam = df.loc[am_filter(df, 'timestamp1', after),:].copy()\n",
    "    dfpm = df.loc[pm_filter(df, 'timestamp1', after),:].copy()\n",
    "#     dfam.sort_values(\"timestamp1\",inplace=True)\n",
    "#     dfam.reset_index(drop=True, inplace=True)\n",
    "#     dfpm.sort_values(\"timestamp1\",inplace=True)\n",
    "#     dfpm.reset_index(drop=True, inplace=True)\n",
    "    return dfam, dfpm\n",
    "\n",
    "# dfam, dfpm = make_deltas_split_ampm(df, drop_outliers=True)\n",
    "# del df\n",
    "# dfyam, dfypm = make_deltas_split_ampm(dfy, drop_outliers=False, after=True)\n",
    "# del dfy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfpm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_y(df):\n",
    "    # prefilter so we only calculate y speeds we need\n",
    "    df.drop(df.index[\n",
    "          ~(bb_filter(df, which=1) &\n",
    "          bb_filter(df, which=2))\n",
    "    ],inplace=True)\n",
    "    return df\n",
    "\n",
    "# dfyam = filter_y(dfyam)\n",
    "# dfypm = filter_y(dfypm)\n",
    "# dfyam.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 7.39 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def calc_speed(df, drop_outliers=True):\n",
    "    # calculate distance and speed based on provided formula\n",
    "    EARTH_RADIUS = 6371*1000  # m\n",
    "    lat1 = np.radians(df['latitude1'])\n",
    "    lat2 = np.radians(df['latitude2'])\n",
    "    lon1 = np.radians(df['longitude1'])\n",
    "    lon2 = np.radians(df['longitude2'])\n",
    "    dlat = np.radians(df['latitude2']-df['latitude1'])\n",
    "    dlon = np.radians(df['longitude2']-df['longitude1'])\n",
    "    a = np.sin(dlat/2) * np.sin(dlat/2) + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2) * np.sin(dlon/2)\n",
    "    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))\n",
    "    df['distance'] = EARTH_RADIUS * c\n",
    "    if drop_outliers:\n",
    "        df.drop(df.index[df['distance'] > D_MAX], inplace=True)\n",
    "    df['speed'] = df['distance'] / (df['timestamp2'] - df['timestamp1']).dt.total_seconds() * 3.6\n",
    "    if drop_outliers:\n",
    "        df.drop(df.index[df['speed'] > V_MAX], inplace=True)\n",
    "    return df\n",
    "\n",
    "# dfam = calc_speed(dfam, drop_outliers=True)\n",
    "# dfpm = calc_speed(dfpm, drop_outliers=True)\n",
    "# dfyam = calc_speed(dfyam, drop_outliers=False)\n",
    "# dfypm = calc_speed(dfypm, drop_outliers=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfam.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filters for northbound or southbound (nb includes zeros as noted by fit in 1_exploration)\n",
    "def y_dir_filter(df):\n",
    "    return (df['latitude2']-df['latitude1']>=0)\n",
    "\n",
    "# filters for northbound (excludes no movement)\n",
    "def nbfilter(df):\n",
    "    return (df['latitude2']-df['latitude1']>0)\n",
    "\n",
    "# filters for southbound (excludes no movement)\n",
    "def sbfilter(df):\n",
    "    return (df['latitude2']-df['latitude1']<0)\n",
    "\n",
    "# filters for no NS movement\n",
    "def nnsfilter(df):\n",
    "    return (df['latitude2']-df['latitude1']==0)\n",
    "\n",
    "# filters for eastbound (excludes no movement)\n",
    "def ebfilter(df):\n",
    "    return (df['longitude2']-df['longitude1']>0)\n",
    "\n",
    "# filters for westbound (excludes no movement)\n",
    "def wbfilter(df):\n",
    "    return (df['longitude2']-df['longitude1']<0)\n",
    "\n",
    "# filters for no EW movement\n",
    "def nwefilter(df):\n",
    "    return (df['latitude2']-df['latitude1']==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index placeholder to ensure full indices\n",
    "def index_placeholder(timecol, am=True, after=False):\n",
    "    start = timecol[0].replace(hour=5,minute=55,second=0) if am else timecol[0].replace(hour=15,minute=55,second=0)\n",
    "    return pd.Series(index=[\n",
    "        (start + i*datetime.timedelta(minutes=5)) for i in range(\n",
    "            5*12+(2 if after else 1)\n",
    "        )])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_y(dfam, dfpm):\n",
    "    \n",
    "    am_placeholder = index_placeholder(dfam['timestamp1'], am=True, after=True)\n",
    "    pm_placeholder = index_placeholder(dfpm['timestamp1'], am=False, after=True)\n",
    "    \n",
    "    amidx = y_dir_filter(dfam)\n",
    "    pmidx = y_dir_filter(dfpm)\n",
    "    dfnam = dfam.loc[amidx,:]\n",
    "    dfsam = dfam.loc[~amidx,:]\n",
    "    dfnpm = dfpm.loc[pmidx,:]\n",
    "    dfspm = dfpm.loc[~pmidx,:]\n",
    "    \n",
    "    dfnam = dfnam.groupby(GPR).mean()['speed'] + NB_CORRECTION\n",
    "    dfsam = dfsam.groupby(GPR).mean()['speed'] + SB_CORRECTION\n",
    "    dfnpm = dfnpm.groupby(GPR).mean()['speed'] + NB_CORRECTION\n",
    "    dfspm = dfspm.groupby(GPR).mean()['speed'] + SB_CORRECTION\n",
    "    \n",
    "    dfam = pd.DataFrame({'p': am_placeholder, 'nb_speed': dfnam, 'sb_speed': dfsam})\n",
    "    dfam.drop('p',axis=1,inplace=True)\n",
    "    dfam.fillna(dfam.max(), inplace=True)\n",
    "    dfpm = pd.DataFrame({'p': pm_placeholder, 'nb_speed': dfnpm, 'sb_speed': dfspm})\n",
    "    dfpm.drop('p',axis=1,inplace=True)\n",
    "    dfpm.fillna(dfpm.max(), inplace=True)\n",
    "    \n",
    "    return dfam, dfpm\n",
    "\n",
    "# dfyam, dfypm = make_y(dfyam, dfypm)\n",
    "# dfyam.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfypm.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# north section above bbox\n",
    "def n(df, which=1): # first lat/long in delta or second?\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] > XMIN) & \\\n",
    "           (df[lon] < XMAX) & \\\n",
    "           (df[lat] > YMAX) & \\\n",
    "           (df[lat] < ABOVE)\n",
    "\n",
    "# south section below bbox\n",
    "def s(df, which=1):\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] > XMIN) & \\\n",
    "           (df[lon] < XMAX) & \\\n",
    "           (df[lat] < YMIN) & \\\n",
    "           (df[lat] > BELOW)\n",
    "\n",
    "# north intersection and up\n",
    "def nn(df, which=1):\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] >= LLEFT) & \\\n",
    "           (df[lon] <= LRGHT) & \\\n",
    "           (df[lat] > ABOVE)\n",
    "\n",
    "# south intersection and below\n",
    "def ss(df, which=1):\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] >= LLEFT) & \\\n",
    "           (df[lon] <= LRGHT) & \\\n",
    "           (df[lat] < BELOW)\n",
    "\n",
    "# western intersection 1 (northern one) left of bbox filter\n",
    "def w(df, which=1):\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] < XMIN) & \\\n",
    "           (df[lon] > WLEFT) & \\\n",
    "           (df[lat] < WNRTH) & \\\n",
    "           (df[lat] > WSOUT)\n",
    "\n",
    "\n",
    "# western intersection 2 (southern one) left of bbox filter\n",
    "def e(df, which=1):\n",
    "    lon, lat = which_lon_lat(which)\n",
    "    return (df[lon] > XMAX) & \\\n",
    "           (df[lon] < ERGHT) & \\\n",
    "           (df[lat] > ENRTH) & \\\n",
    "           (df[lat] < ESOUT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_x(dfam, dfpm):\n",
    "    \n",
    "    am_placeholder = index_placeholder(dfam['timestamp1'], am=True, after=False)\n",
    "    pm_placeholder = index_placeholder(dfpm['timestamp1'], am=False, after=False)\n",
    "    \n",
    "    for i,df in enumerate([dfam, dfpm]):\n",
    "        \n",
    "        ### NORTHBOUND LINKS\n",
    "        \n",
    "        # n_n_nb: N -> N northbound\n",
    "        nonzeros = df.loc[n(df,1) & n(df,2) & nbfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        n_n_zeros_count = df.loc[n(df,1) & n(df,2) & nnsfilter(df),:].groupby(GPR).count()['speed']/2 # apply half of zeros to NB, other half to SB\n",
    "        n_n_nb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + n_n_zeros_count)\n",
    "        \n",
    "        # n_nn: N -> NN\n",
    "        n_nn = df.loc[n(df,1) & nn(df,2),:].groupby(GPR).mean()['speed']\n",
    "        \n",
    "        # s_n: S -> N\n",
    "        s_n = df.loc[s(df,1) & n(df,2),:].groupby(GPR).mean()['speed']\n",
    "        \n",
    "        # s_s_nb: S -> S northbound\n",
    "        nonzeros = df.loc[s(df,1) & s(df,2) & nbfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        s_s_zeros_count = df.loc[s(df,1) & s(df,2) & nnsfilter(df),:].groupby(GPR).count()['speed']/2\n",
    "        s_s_nb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + s_s_zeros_count)\n",
    "        \n",
    "        # ss_s: SS -> S\n",
    "        ss_s = df.loc[ss(df,1) & s(df,2),:].groupby(GPR).mean()['speed']\n",
    "        \n",
    "        \n",
    "        ### SOUTHBOUND LINKS\n",
    "        \n",
    "        # n_n_sb: N -> N southbound\n",
    "        nonzeros = df.loc[n(df,1) & n(df,2) & sbfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        n_n_sb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + n_n_zeros_count)\n",
    "        \n",
    "        # nn_n: NN -> N\n",
    "        nn_n = df.loc[nn(df,1) & n(df,2),:].groupby(GPR).mean()['speed']\n",
    "        \n",
    "        # n_s: N -> S\n",
    "        n_s = df.loc[n(df,1) & s(df,2),:].groupby(GPR).mean()['speed']\n",
    "        \n",
    "        # s_s_sb: S -> S southbound\n",
    "        nonzeros = df.loc[s(df,1) & s(df,2) & sbfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        s_s_sb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + s_s_zeros_count)\n",
    "        \n",
    "        # s_ss: S -> SS\n",
    "        s_ss = df.loc[s(df,1) & ss(df,2),:].groupby(GPR).mean()['speed']\n",
    "        \n",
    "        \n",
    "        ### MID LINKS\n",
    "        \n",
    "        # w_w_eb: W -> W eastbound\n",
    "        nonzeros = df.loc[w(df,1) & w(df,2) & ebfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        zeros_count = df.loc[w(df,1) & w(df,2) & nwefilter(df),:].groupby(GPR).count()['speed']/2\n",
    "        w_w_eb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + zeros_count)\n",
    "        \n",
    "        # w_w_wb: W -> W westbound\n",
    "        nonzeros = df.loc[w(df,1) & w(df,2) & wbfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        w_w_wb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + zeros_count)\n",
    "        \n",
    "        # e_e_eb: E -> E eastbound\n",
    "        nonzeros = df.loc[e(df,1) & e(df,2) & ebfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        zeros_count = df.loc[e(df,1) & e(df,2) & nwefilter(df),:].groupby(GPR).count()['speed']/2\n",
    "        e_e_eb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + zeros_count)\n",
    "        \n",
    "        # e_e_wb: E -> E westbound\n",
    "        nonzeros = df.loc[e(df,1) & e(df,2) & wbfilter(df),:].groupby(GPR)\n",
    "        nonzeros_count = nonzeros.count()['speed']\n",
    "        e_e_wb = nonzeros.mean()['speed']*nonzeros_count/(nonzeros_count + zeros_count)\n",
    "        \n",
    "        \n",
    "        ### log\n",
    "        \n",
    "        if i==0: # AM\n",
    "            dfnam = pd.DataFrame({\n",
    "                'p': am_placeholder,\n",
    "                'n_n_nb': n_n_nb,\n",
    "                'n_nn': n_nn,\n",
    "                's_n': s_n,\n",
    "                's_s_nb': s_s_nb,\n",
    "                'ss_s': ss_s\n",
    "            })\n",
    "            dfnam.drop('p',axis=1,inplace=True)\n",
    "            dfnam.fillna(dfnam.max(), inplace=True)\n",
    "            dfsam = pd.DataFrame({\n",
    "                'p': am_placeholder,\n",
    "                'n_n_sb': n_n_sb,\n",
    "                'nn_n': nn_n,\n",
    "                'n_s': n_s,\n",
    "                's_s_sb': s_s_sb,\n",
    "                's_ss': s_ss\n",
    "            })\n",
    "            dfsam.drop('p',axis=1,inplace=True)\n",
    "            dfsam.fillna(dfsam.max(), inplace=True)\n",
    "            dfewam = pd.DataFrame({\n",
    "                'p': am_placeholder,\n",
    "                'w_w_eb': w_w_eb,\n",
    "                'w_w_wb': w_w_wb,\n",
    "                'e_e_eb': e_e_eb,\n",
    "                'e_e_wb': e_e_wb\n",
    "            })\n",
    "            dfewam.drop('p',axis=1,inplace=True)\n",
    "            dfewam.fillna(dfewam.max(), inplace=True)\n",
    "        else: # PM\n",
    "            dfnpm = pd.DataFrame({\n",
    "                'p': pm_placeholder,\n",
    "                'n_n_nb': n_n_nb,\n",
    "                'n_nn': n_nn,\n",
    "                's_n': s_n,\n",
    "                's_s_nb': s_s_nb,\n",
    "                'ss_s': ss_s\n",
    "            })\n",
    "            dfnpm.drop('p',axis=1,inplace=True)\n",
    "            dfnpm.fillna(dfnpm.max(), inplace=True)\n",
    "            dfspm = pd.DataFrame({\n",
    "                'p': pm_placeholder,\n",
    "                'n_n_sb': n_n_sb,\n",
    "                'nn_n': nn_n,\n",
    "                'n_s': n_s,\n",
    "                's_s_sb': s_s_sb,\n",
    "                's_ss': s_ss\n",
    "            })\n",
    "            dfspm.drop('p',axis=1,inplace=True)\n",
    "            dfspm.fillna(dfspm.max(), inplace=True)\n",
    "            dfewpm = pd.DataFrame({\n",
    "                'p': pm_placeholder,\n",
    "                'w_w_eb': w_w_eb,\n",
    "                'w_w_wb': w_w_wb,\n",
    "                'e_e_eb': e_e_eb,\n",
    "                'e_e_wb': e_e_wb\n",
    "            })\n",
    "            dfewpm.drop('p',axis=1,inplace=True)\n",
    "            dfewpm.fillna(dfewpm.max(), inplace=True)\n",
    "            \n",
    "    return dfnam, dfsam, dfewam, dfnpm, dfspm, dfewpm\n",
    "\n",
    "# dfnam, dfsam, dfewam, dfnpm, dfspm, dfewpm = make_x(dfam, dfpm)\n",
    "# dfewpm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# del dfam, dfpm, dfnam, dfsam, dfewam, dfnpm, dfspm, dfewpm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 0 ns, total: 0 ns\n",
      "Wall time: 8.11 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def process_date(date, make_y_df=True, has_header=False):\n",
    "    dfy = read_gps('data/xian/gps_2016{}'.format(date[0]+date[1]), has_header)\n",
    "    df = delete_bbox(dfy)\n",
    "    if make_y_df:\n",
    "        dfyam, dfypm = make_deltas_split_ampm(dfy, drop_outliers=False, after=True)\n",
    "        del dfy\n",
    "        dfyam = filter_y(dfyam)\n",
    "        dfypm = filter_y(dfypm)\n",
    "        dfyam = calc_speed(dfyam, drop_outliers=False)\n",
    "        dfypm = calc_speed(dfypm, drop_outliers=False)\n",
    "        dfyam, dfypm = make_y(dfyam, dfypm)\n",
    "        dfyam.to_pickle('data/processed/{}-{}_y_am.pkl'.format(date[0],date[1]))\n",
    "        del dfyam\n",
    "        dfypm.to_pickle('data/processed/{}-{}_y_pm.pkl'.format(date[0],date[1]))\n",
    "        del dfypm\n",
    "    else:\n",
    "        del dfy\n",
    "    dfam, dfpm = make_deltas_split_ampm(df, drop_outliers=True)\n",
    "    del df\n",
    "    dfam = calc_speed(dfam, drop_outliers=True)\n",
    "    dfpm = calc_speed(dfpm, drop_outliers=True)\n",
    "    dfnam, dfsam, dfewam, dfnpm, dfspm, dfewpm = make_x(dfam, dfpm)\n",
    "    dfnam.to_pickle('data/processed/{}-{}_n_am.pkl'.format(date[0],date[1]))\n",
    "    del dfnam\n",
    "    dfsam.to_pickle('data/processed/{}-{}_s_am.pkl'.format(date[0],date[1]))\n",
    "    del dfsam\n",
    "    dfewam.to_pickle('data/processed/{}-{}_ew_am.pkl'.format(date[0],date[1]))\n",
    "    del dfewam\n",
    "    dfnpm.to_pickle('data/processed/{}-{}_n_pm.pkl'.format(date[0],date[1]))\n",
    "    del dfnpm\n",
    "    dfspm.to_pickle('data/processed/{}-{}_s_pm.pkl'.format(date[0],date[1]))\n",
    "    del dfspm\n",
    "    dfewpm.to_pickle('data/processed/{}-{}_ew_pm.pkl'.format(date[0],date[1]))\n",
    "    del dfewpm\n",
    "\n",
    "# process_date(('10','10'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# for filename in [('10',date) for date in oct_mwf+oct_thu] + [('11',date) for date in nov_mwf+nov_thu]:\n",
    "#     process_date(filename)\n",
    "process_date(('12','01'), make_y_df=True, has_header=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}