feature_eng_2_first_time_seen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = {\n",
    "    'ip': 'uint32',\n",
    "    'app': 'uint16',\n",
    "    'device': 'uint16',\n",
    "    'os': 'uint16',\n",
    "    'channel': 'uint16',\n",
    "    'is_attributed': 'int8'\n",
    "}\n",
    "\n",
    "to_read = ['ip','click_time'] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Min IP Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tmp = pd.read_csv('./data/train.csv.zip', \n",
    "                        usecols=to_read, \n",
    "                        dtype=dtypes, \n",
    "                        parse_dates=['click_time'], \n",
    "                        chunksize=10**6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ip_time = pd.DataFrame()\n",
    "\n",
    "for df in train_tmp:\n",
    "    \n",
    "    agg = df.groupby('ip')['click_time'].min().reset_index()\n",
    "    \n",
    "    min_ip_time = pd.concat([min_ip_time, agg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ip_time_train = min_ip_time.groupby('ip')['click_time'].min().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(277396, 2)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_ip_time_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "734"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train_tmp, min_ip_time\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Min IP Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tmp = pd.read_csv('./data/test_supplement.csv.zip', \n",
    "                        usecols=to_read, \n",
    "                        dtype=dtypes, \n",
    "                        parse_dates=['click_time'], \n",
    "                        chunksize=10**6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ip_time = pd.DataFrame()\n",
    "\n",
    "for df in test_tmp:\n",
    "    \n",
    "    agg = df.groupby('ip')['click_time'].min().reset_index()\n",
    "    \n",
    "    min_ip_time = pd.concat([min_ip_time, agg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ip_time_test = min_ip_time.groupby('ip')['click_time'].min().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(126414, 2)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_ip_time_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ip_time = pd.concat([min_ip_time_train, min_ip_time_test]).groupby('ip')['click_time'].min().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(364779, 2)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_ip_time.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "350"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del test_tmp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ip_time.rename(columns={'click_time':'ip_first_seen'}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use as feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('./data/train.csv.zip', skiprows=range(1,144903891), nrows=40000000, \n",
    "                       parse_dates=['click_time'], usecols=to_read, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('./data/test.csv.zip', usecols=to_read, parse_dates=['click_time'], dtype=dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df = pd.concat([train_df,test_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del train_df, test_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df = concat_df.merge(min_ip_time, on='ip', how='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate Time to First Seen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df['time_diff_first_seen'] = (concat_df['click_time'] - concat_df['ip_first_seen']).astype('timedelta64[s]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df['log2_time_diff_first_seen'] = np.log2(1 + concat_df['time_diff_first_seen'].values).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ip                                   uint32\n",
       "click_time                   datetime64[ns]\n",
       "ip_first_seen                datetime64[ns]\n",
       "time_diff_first_seen                float64\n",
       "log2_time_diff_first_seen             int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concat_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df.to_csv('tmp/time_diff.csv.gzip', index=False, chunksize=10**6, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}