feature_eng_1_next_click.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import lightgbm as lgb\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from contextlib import contextmanager\n",
    "\n",
    "@contextmanager\n",
    "def timer(name):\n",
    "    t0 = time.time()\n",
    "    yield\n",
    "    print(f'[{name}] done in {time.time() - t0:.0f} s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_add_counts(df, cols):\n",
    "    arr_slice = df[cols].values\n",
    "    unq, unqtags, counts = np.unique(np.ravel_multi_index(arr_slice.T, arr_slice.max(axis=0) + 1),\n",
    "                                     return_inverse=True, return_counts=True)\n",
    "    df[\"_\".join(cols)+'_count'] = counts[unqtags]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_count_features(df):\n",
    "    with timer(\"add count features\"):\n",
    "        df['click_time']= pd.to_datetime(df['click_time'])\n",
    "        dt= df['click_time'].dt\n",
    "        df['day'] = dt.day.astype('uint8')\n",
    "        df['hour'] = dt.hour.astype('uint8')\n",
    "        df['minute'] = dt.minute.astype('uint8')\n",
    "        del(dt)\n",
    "        \n",
    "        df_add_counts(df, ['ip'])\n",
    "        df_add_counts(df, ['ip','day','hour','minute'])\n",
    "        df_add_counts(df, ['os','device'])\n",
    "        df_add_counts(df, ['os','app','channel'])\n",
    "        \n",
    "        df_add_counts(df, ['ip', 'day', 'hour'])\n",
    "        df_add_counts(df, ['ip', 'app'])\n",
    "        df_add_counts(df, ['ip', 'app', 'os'])\n",
    "        df_add_counts(df, ['ip', 'device'])\n",
    "        df_add_counts(df, ['app', 'channel'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_click_features(df):\n",
    "    D= 2**26\n",
    "    df['category'] = (df['ip'].astype(str) + \"_\" + df['app'].astype(str) + \"_\" + df['device'].astype(str) \\\n",
    "                      + \"_\" + df['os'].astype(str)).apply(hash) % D\n",
    "    \n",
    "    df['epochtime']= df['click_time'].astype(np.int64) // 10 ** 9\n",
    "        \n",
    "    with timer(\"Adding next click times\"):\n",
    "        \n",
    "        click_buffer= np.full(D, 3000000000, dtype=np.uint32)\n",
    "        next_clicks= []\n",
    "        for category, time in zip(reversed(df['category'].values), reversed(df['epochtime'].values)):\n",
    "            next_clicks.append(click_buffer[category]-time)\n",
    "            click_buffer[category]= time\n",
    "        del(click_buffer)\n",
    "        df['next_click']= list(reversed(next_clicks))\n",
    "   \n",
    "    with timer(\"Adding previous click times\"):\n",
    "        \n",
    "        click_buffer= np.zeros(D, dtype=np.uint32)\n",
    "        prev_clicks= []\n",
    "        for category, time in zip(df['category'].values, df['epochtime'].values):\n",
    "            prev_clicks.append(time-click_buffer[category])\n",
    "            click_buffer[category]= time\n",
    "        del(click_buffer)\n",
    "        df['prev_click']= prev_clicks\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "613739358056957992"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hash('da_rou_mao')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_bin_feature(df, features):\n",
    "    with timer(\"Log-binning features\"):\n",
    "        for fea in features: \n",
    "            df[fea]= np.log2(1 + df[fea].values).astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = {\n",
    "    'ip': 'uint32',\n",
    "    'app': 'uint16',\n",
    "    'device': 'uint16',\n",
    "    'os': 'uint16',\n",
    "    'channel': 'uint16',\n",
    "    'is_attributed': 'int8'\n",
    "}\n",
    "\n",
    "to_read = ['ip','app','device','os','channel','click_time','is_attributed'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_tmp = pd.read_csv('./data/train.csv.zip', parse_dates=['click_time'], \n",
    "                           usecols=to_read, dtype=dtypes, chunksize = 10**6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.concat(train_df_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train_df_tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df_tmp = pd.read_csv('./data/test.csv.zip', parse_dates=['click_time'], dtype=dtypes, chunksize = 10**6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.concat(test_df_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "126"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del test_df_tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_train = train_df.shape[0]\n",
    "y = train_df.is_attributed.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(184903890, 7)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18790469, 7)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Concat Train & Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_column = ['ip','app','device','os','channel','click_time']\n",
    "concat_df = pd.concat([train_df[common_column],test_df[common_column]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train_df, test_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[add count features] done in 1215 s\n"
     ]
    }
   ],
   "source": [
    "make_count_features(concat_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Adding next click times] done in 685 s\n",
      "[Adding previous click times] done in 225 s\n"
     ]
    }
   ],
   "source": [
    "make_click_features(concat_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ip                                  uint32\n",
       "app                                 uint16\n",
       "device                              uint16\n",
       "os                                  uint16\n",
       "channel                             uint16\n",
       "click_time                  datetime64[ns]\n",
       "day                                  uint8\n",
       "hour                                 uint8\n",
       "minute                               uint8\n",
       "ip_count                             int64\n",
       "ip_day_hour_minute_count             int64\n",
       "os_device_count                      int64\n",
       "os_app_channel_count                 int64\n",
       "ip_day_hour_count                    int64\n",
       "ip_app_count                         int64\n",
       "ip_app_os_count                      int64\n",
       "ip_device_count                      int64\n",
       "app_channel_count                    int64\n",
       "category                             int64\n",
       "epochtime                            int64\n",
       "next_click                           int64\n",
       "prev_click                           int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['ip_count',\n",
    "            'ip_day_hour_minute_count',\n",
    "            'os_device_count',\n",
    "            'os_app_channel_count',\n",
    "            'ip_day_hour_count',\n",
    "            'ip_app_count',\n",
    "            'ip_app_os_count',\n",
    "            'ip_device_count',\n",
    "            'app_channel_count',\n",
    "            'next_click',\n",
    "            'prev_click']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Log-binning features] done in 134 s\n"
     ]
    }
   ],
   "source": [
    "log_bin_feature(concat_df, features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save concat_df to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_to_write = [col for col in list(concat_df) if col not in ['click_time','category','epochtime']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df.to_csv('tmp/concat_df.csv.gz', chunksize=10**6, compression='gzip', index=False, \n",
    "                 columns = col_to_write)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "features = ['ip_count',\n",
    "            'ip_day_hour_minute_count',\n",
    "            'os_device_count',\n",
    "            'os_app_channel_count',\n",
    "            'ip_day_hour_count',\n",
    "            'ip_app_count',\n",
    "            'ip_app_os_count',\n",
    "            'ip_device_count',\n",
    "            'app_channel_count',\n",
    "            'next_click',\n",
    "            'prev_click']\n",
    "\n",
    "log_bin_feature(concat_df, features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = \"is_attributed\"\n",
    "categorical_features = ['ip','app','os','channel','device']\n",
    "predictors = list(set(concat_df.columns)-set([target])-set(['click_time','category','epochtime']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgbtrain = lgb.Dataset(concat_df.iloc[:num_train][predictors].values, label=y,\n",
    "                      feature_name=predictors,\n",
    "                      categorical_feature=categorical_features\n",
    "                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'subsample_for_bin': 200000,  # Number of samples for constructing bin\n",
    "    'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n",
    "    'reg_alpha': 0,  # L1 regularization term on weights\n",
    "    'reg_lambda': 0,  # L2 regularization term on weights\n",
    "    'nthread': 4,\n",
    "    'verbose': 0,\n",
    "    'metric':'auc',     \n",
    " \n",
    "    'learning_rate': 0.15,\n",
    "    'num_leaves': 7,  # 2^max_depth - 1\n",
    "    'max_depth': 3,  # -1 means no limit\n",
    "    'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n",
    "    'max_bin': 100,  # Number of bucketed bin for feature values\n",
    "    'subsample': 0.7,  # Subsample ratio of the training instance.\n",
    "    'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n",
    "    'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.\n",
    "    'min_child_weight': 0,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n",
    "    'scale_pos_weight':99\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evals_results = {}\n",
    "num_boost_round = 200\n",
    "early_stopping_rounds = 30\n",
    "\n",
    "booster = lgb.train(\n",
    "     lgb_params, \n",
    "     lgbtrain, \n",
    "     valid_sets=[lgbtrain], \n",
    "     valid_names=['train'], \n",
    "     evals_result=evals_results, \n",
    "     num_boost_round=num_boost_round,\n",
    "     early_stopping_rounds=early_stopping_rounds,\n",
    "     verbose_eval=1\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_pred(model, df, batch_size=10**5):\n",
    "    \n",
    "    N_batch = int(df.shape[0]/batch_size)\n",
    "    pred = []\n",
    "    \n",
    "    for i in range(N_batch+1):\n",
    "        if i%100 == 0 & i>0:\n",
    "            print(\"finished batch %d/%d\" %(i, N_batch))\n",
    "        \n",
    "        batch_X = df.iloc[i*batch_size : (i+1)*batch_size] \n",
    "        batch_pred = model.predict(batch_X.values)\n",
    "        pred.extend(list(batch_pred))\n",
    "    \n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preditions = batch_pred(booster, concat_df.iloc[num_train: ][predictors], batch_size=10**5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(preditions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('data/sample_submission.csv.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub['is_attributed'] =  preditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}