feature_eng_4_isolation_forest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://stats.stackexchange.com/questions/312619/does-isolation-forest-need-an-anomaly-sample-during-training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### data 1: concat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = {  \n",
    "            'ip'            : 'uint32',\n",
    "            'app'           : 'uint16',\n",
    "            'device'        : 'uint16',\n",
    "            'os'            : 'uint16',\n",
    "            'channel'       : 'uint16',\n",
    "            'day'           : 'uint8',\n",
    "            'hour'          : 'uint8',\n",
    "            'minute'        : 'uint8',\n",
    "            'ip_count'      : 'uint32',\n",
    "            'ip_day_hour_minute_count' : 'uint32',\n",
    "            'os_device_count'          : 'uint32',\n",
    "            'os_app_channel_count'     : 'uint32',\n",
    "            #'ip_day_hour_count'        : 'uint32',\n",
    "            #'ip_app_count'             : 'uint32',\n",
    "            #'ip_app_os_count'          : 'uint32',\n",
    "            'ip_device_count'          : 'uint32',\n",
    "            'app_channel_count'        : 'uint32',\n",
    "            'next_click'               : 'int64',\n",
    "            'prev_click'               : 'int64'        \n",
    "            }\n",
    "\n",
    "to_read = ['ip', 'app', 'device', 'os', 'channel', 'day', 'hour', 'minute',\n",
    "            'ip_count', 'ip_day_hour_minute_count',\n",
    "            'os_device_count',\n",
    "            'os_app_channel_count',\n",
    "            #'ip_day_hour_count',\n",
    "            #'ip_app_count',\n",
    "            #'ip_app_os_count',\n",
    "            'ip_device_count',\n",
    "            'app_channel_count',\n",
    "            'next_click',\n",
    "            'prev_click'\n",
    "          ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df_tmp = pd.read_csv('tmp/concat_df.csv.gz', compression='gzip', dtype=dtypes, usecols=to_read, chunksize = 10**6)\n",
    "concat_df = pd.concat(concat_df_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "406"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del concat_df_tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### data 2: time feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_feature = pd.read_csv('tmp/time_diff.csv.gz', compression='gzip', usecols=['log2_time_diff_first_seen'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### data 3: agg future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_dtype = {\n",
    "    'nuniq_ip_channel':            'uint32',\n",
    "    'cumcnt_ip_device_os_app':     'uint32',\n",
    "    'nuniq_ip_day_hour':           'uint32',\n",
    "    'nuniq_ip_app':                'uint32',\n",
    "    'nuniq_ip_app_os':             'uint32',\n",
    "    'nuniq_ip_device':             'uint32',\n",
    "    'nuniq_app_channel':           'uint32',\n",
    "    'cumcnt_ip_os':                'uint32',\n",
    "    'nuniq_ip_device_os_app':      'uint32',\n",
    "    'ip_tcount':                   'uint32',\n",
    "    'ip_app_count':                'uint32',\n",
    "    'ip_app_os_count':             'uint32',\n",
    "    'ip_tchan_count':              'float32',\n",
    "    'ip_app_os_var':               'float32',\n",
    "    'ip_app_channel_var_day':      'float32',\n",
    "    'ip_app_channel_mean_hour':    'float32'\n",
    "}\n",
    "\n",
    "agg_to_read = ['nuniq_ip_channel', \n",
    "               'cumcnt_ip_device_os_app',\n",
    "               'nuniq_ip_day_hour',\n",
    "               'nuniq_ip_app',\n",
    "               'nuniq_ip_app_os',\n",
    "               'nuniq_ip_device',\n",
    "               'nuniq_app_channel',\n",
    "               'cumcnt_ip_os',\n",
    "               'nuniq_ip_device_os_app',\n",
    "               'ip_tcount',\n",
    "               'ip_app_count',\n",
    "               'ip_app_os_count',\n",
    "               'ip_tchan_count',\n",
    "               'ip_app_os_var',\n",
    "               'ip_app_channel_var_day',\n",
    "               'ip_app_channel_mean_hour']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_feature_tmp = pd.read_csv('tmp/agg_features.csv.gz', compression='gzip', dtype=agg_dtype, \n",
    "                              usecols=agg_to_read, chunksize=10**6)\n",
    "\n",
    "agg_feature = pd.concat(agg_feature_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "406"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del agg_feature_tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "fill_agg = ['ip_tchan_count','ip_app_os_var','ip_app_channel_var_day','ip_app_channel_mean_hour']\n",
    "agg_feature[fill_agg] = agg_feature[fill_agg].fillna(agg_feature[fill_agg].mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Concat "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df = pd.concat([concat_df, time_feature, agg_feature], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del time_feature, agg_feature\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = pd.read_csv('tmp/train_y.csv.gz', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "nrows = 40000000\n",
    "n_sample = 5*10**5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "if_idx = train_y.loc[train_y['is_attributed']==0].sample(n_sample, random_state=1).index.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate log2 feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_bin_feature(df, features):\n",
    "    for fea in features: \n",
    "        df[fea]= np.log2(1 + df[fea].values).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['ip_count',\n",
    "            'ip_day_hour_minute_count',\n",
    "            'os_device_count',\n",
    "            'os_app_channel_count',\n",
    "            'ip_tcount',\n",
    "            'ip_app_count',\n",
    "            'ip_app_os_count',\n",
    "            'ip_device_count',\n",
    "            'app_channel_count',\n",
    "            'next_click',\n",
    "            'prev_click',\n",
    "            'nuniq_ip_channel', \n",
    "            'cumcnt_ip_device_os_app',\n",
    "            'nuniq_ip_day_hour',\n",
    "            'nuniq_ip_app',\n",
    "            'nuniq_ip_app_os',\n",
    "            'nuniq_ip_device',\n",
    "            'nuniq_app_channel',\n",
    "            'cumcnt_ip_os',\n",
    "            'nuniq_ip_device_os_app']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_bin_feature(concat_df, features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_if = concat_df.iloc[if_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate feature list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictors = features + ['hour', 'minute', 'os', 'app', 'device', 'log2_time_diff_first_seen']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Isolation Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = IsolationForest(n_estimators = 110,\n",
    "                         contamination = 0.0025,\n",
    "                         random_state = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IsolationForest(bootstrap=False, contamination=0.0025, max_features=1.0,\n",
       "        max_samples='auto', n_estimators=110, n_jobs=1, random_state=1,\n",
       "        verbose=0)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest.fit(train_if[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = concat_df.iloc[:nrows][predictors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_pred(model, X, batch_size=100000):\n",
    "    \n",
    "    N_batch = int(X.shape[0]/batch_size)\n",
    "    pred = []\n",
    "    \n",
    "    for i in range(N_batch+1):\n",
    "        if i%10 == 0 and i>0:\n",
    "            print(\"finished batch %d/%d\" %(i, N_batch))\n",
    "        \n",
    "        batch_X = X.iloc[i*batch_size : (i+1)*batch_size] \n",
    "        \n",
    "        if batch_X.shape[0] > 0:\n",
    "            batch_pred = -model.decision_function(batch_X)\n",
    "            pred.extend(list(batch_pred))\n",
    "    \n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished batch 10/400\n",
      "finished batch 20/400\n",
      "finished batch 30/400\n",
      "finished batch 40/400\n",
      "finished batch 50/400\n",
      "finished batch 60/400\n",
      "finished batch 70/400\n",
      "finished batch 80/400\n",
      "finished batch 90/400\n",
      "finished batch 100/400\n",
      "finished batch 110/400\n",
      "finished batch 120/400\n",
      "finished batch 130/400\n",
      "finished batch 140/400\n",
      "finished batch 150/400\n",
      "finished batch 160/400\n",
      "finished batch 170/400\n",
      "finished batch 180/400\n",
      "finished batch 190/400\n",
      "finished batch 200/400\n",
      "finished batch 210/400\n",
      "finished batch 220/400\n",
      "finished batch 230/400\n",
      "finished batch 240/400\n",
      "finished batch 250/400\n",
      "finished batch 260/400\n",
      "finished batch 270/400\n",
      "finished batch 280/400\n",
      "finished batch 290/400\n",
      "finished batch 300/400\n",
      "finished batch 310/400\n",
      "finished batch 320/400\n",
      "finished batch 330/400\n",
      "finished batch 340/400\n",
      "finished batch 350/400\n",
      "finished batch 360/400\n",
      "finished batch 370/400\n",
      "finished batch 380/400\n",
      "finished batch 390/400\n",
      "finished batch 400/400\n"
     ]
    }
   ],
   "source": [
    "if_pred = batch_pred(forest, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc = roc_auc_score(train_y['is_attributed'], if_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9012363032204544"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del test, if_pred\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished batch 10/587\n",
      "finished batch 20/587\n",
      "finished batch 30/587\n",
      "finished batch 40/587\n",
      "finished batch 50/587\n",
      "finished batch 60/587\n",
      "finished batch 70/587\n",
      "finished batch 80/587\n",
      "finished batch 90/587\n",
      "finished batch 100/587\n",
      "finished batch 110/587\n",
      "finished batch 120/587\n",
      "finished batch 130/587\n",
      "finished batch 140/587\n",
      "finished batch 150/587\n",
      "finished batch 160/587\n",
      "finished batch 170/587\n",
      "finished batch 180/587\n",
      "finished batch 190/587\n",
      "finished batch 200/587\n",
      "finished batch 210/587\n",
      "finished batch 220/587\n",
      "finished batch 230/587\n",
      "finished batch 240/587\n",
      "finished batch 250/587\n",
      "finished batch 260/587\n",
      "finished batch 270/587\n",
      "finished batch 280/587\n",
      "finished batch 290/587\n",
      "finished batch 300/587\n",
      "finished batch 310/587\n",
      "finished batch 320/587\n",
      "finished batch 330/587\n",
      "finished batch 340/587\n",
      "finished batch 350/587\n",
      "finished batch 360/587\n",
      "finished batch 370/587\n",
      "finished batch 380/587\n",
      "finished batch 390/587\n",
      "finished batch 400/587\n",
      "finished batch 410/587\n",
      "finished batch 420/587\n",
      "finished batch 430/587\n",
      "finished batch 440/587\n",
      "finished batch 450/587\n",
      "finished batch 460/587\n",
      "finished batch 470/587\n",
      "finished batch 480/587\n",
      "finished batch 490/587\n",
      "finished batch 500/587\n",
      "finished batch 510/587\n",
      "finished batch 520/587\n",
      "finished batch 530/587\n",
      "finished batch 540/587\n",
      "finished batch 550/587\n",
      "finished batch 560/587\n",
      "finished batch 570/587\n",
      "finished batch 580/587\n"
     ]
    }
   ],
   "source": [
    "if_score = batch_pred(forest, concat_df[predictors], batch_size=100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58790469"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(if_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "if_output = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "if_output['if_score'] = if_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "if_output.to_csv('tmp/if_score_v2.csv.gz', float_format='%.8f', index=False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}