Dask research .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## chunk create time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# import pandas as pd\n",
    "# pandas_df=pd.read_csv(\"dataset.csv\",chunksize=29999)\n",
    "# file_no=1\n",
    "# for df in pandas_df:\n",
    "#     df.to_csv(str(file_no)+\".csv\")\n",
    "#     file_no+=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. DataFrame Comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 Reading File"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.1 Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 560 ms, sys: 66 ms, total: 626 ms\n",
      "Wall time: 626 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "import pandas as pd\n",
    "pandas_df=pd.read_csv(\"dataset.csv\")\n",
    "type(pandas_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.2 dask dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 34.1 ms, sys: 3.54 ms, total: 37.6 ms\n",
      "Wall time: 41.8 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "dask.dataframe.core.DataFrame"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "import dask.dataframe as dd\n",
    "dask_df=dd.read_csv(\"dataset.csv\")\n",
    "type(dask_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.3 Multi file pandas reading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 669 ms, sys: 140 ms, total: 809 ms\n",
      "Wall time: 809 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import os\n",
    "import glob\n",
    "path = r'chunks'                    \n",
    "all_files = glob.glob(os.path.join(path, \"*.csv\"))\n",
    "df_from_each_file = (pd.read_csv(f) for f in all_files)\n",
    "concatenated_df   = pd.concat(df_from_each_file, ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1.4 Multi file in dask dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36.1 ms, sys: 3.95 ms, total: 40 ms\n",
      "Wall time: 40 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import dask.dataframe as dd\n",
    "df = dd.read_csv(os.path.join(r'chunks', '*.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Writting Files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2.1 Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.3 s, sys: 66.3 ms, total: 9.36 s\n",
      "Wall time: 9.37 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pandas_df.to_csv(\"sample1.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2.2 dask dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.1 s, sys: 166 ms, total: 10.2 s\n",
      "Wall time: 10.5 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/home/hauntedsoul/Downloads/dask-tutorial-scipy-2018-master/stats/sample.csv/0.part']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "dask_df.to_csv(\"sample.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 DataFrame Operation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.3.1 Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1.3.1.1 Mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15.4 ms, sys: 0 ns, total: 15.4 ms\n",
      "Wall time: 18.2 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nom_1\n",
       "0    5945.734459\n",
       "1    5950.621687\n",
       "2    5976.289594\n",
       "3    5942.845329\n",
       "4    5964.401518\n",
       "5    6000.618657\n",
       "Name: nom_9, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "pandas_df.groupby('nom_1').nom_9.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1.3.1.2 Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 421 ms, sys: 55.6 ms, total: 477 ms\n",
      "Wall time: 476 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>bin_0</th>\n",
       "      <th>bin_1</th>\n",
       "      <th>bin_2</th>\n",
       "      <th>bin_3</th>\n",
       "      <th>bin_4</th>\n",
       "      <th>nom_0</th>\n",
       "      <th>nom_1</th>\n",
       "      <th>nom_2</th>\n",
       "      <th>...</th>\n",
       "      <th>nom_9</th>\n",
       "      <th>ord_0</th>\n",
       "      <th>ord_1</th>\n",
       "      <th>ord_2</th>\n",
       "      <th>ord_3</th>\n",
       "      <th>ord_4</th>\n",
       "      <th>ord_5</th>\n",
       "      <th>day</th>\n",
       "      <th>month</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1038.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299994</td>\n",
       "      <td>299994</td>\n",
       "      <td>299995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299995</td>\n",
       "      <td>299995</td>\n",
       "      <td>299996</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299996</td>\n",
       "      <td>299996</td>\n",
       "      <td>299997</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299997</td>\n",
       "      <td>299997</td>\n",
       "      <td>299998</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>299998</td>\n",
       "      <td>299998</td>\n",
       "      <td>299999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>299999 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Unnamed: 0      id  bin_0  bin_1  bin_2  bin_3  bin_4  nom_0  nom_1  \\\n",
       "0                0       0    0.0    0.0    0.0    1.0    1.0    1.0    4.0   \n",
       "1                1       1    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "2                2       2    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3                3       3    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4                4       4    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "...            ...     ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "299994      299994  299995    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "299995      299995  299996    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "299996      299996  299997    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "299997      299997  299998    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "299998      299998  299999    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "\n",
       "        nom_2  ...   nom_9  ord_0  ord_1  ord_2  ord_3  ord_4  ord_5  day  \\\n",
       "0         4.0  ...  1038.0    1.0    4.0    2.0    6.0   11.0  151.0  2.0   \n",
       "1         NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "2         NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "3         NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "4         NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "...       ...  ...     ...    ...    ...    ...    ...    ...    ...  ...   \n",
       "299994    NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "299995    NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "299996    NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "299997    NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "299998    NaN  ...     NaN    NaN    NaN    NaN    NaN    NaN    NaN  NaN   \n",
       "\n",
       "        month  target  \n",
       "0         2.0     0.0  \n",
       "1         NaN     NaN  \n",
       "2         NaN     NaN  \n",
       "3         NaN     NaN  \n",
       "4         NaN     NaN  \n",
       "...       ...     ...  \n",
       "299994    NaN     NaN  \n",
       "299995    NaN     NaN  \n",
       "299996    NaN     NaN  \n",
       "299997    NaN     NaN  \n",
       "299998    NaN     NaN  \n",
       "\n",
       "[299999 rows x 26 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "pandas_df.mode(axis = 0) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1.3.1.3 find"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.2 ms, sys: 4.52 ms, total: 15.7 ms\n",
      "Wall time: 14.4 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16        0\n",
       "19        0\n",
       "28        2\n",
       "48        4\n",
       "50        2\n",
       "         ..\n",
       "299894    1\n",
       "299954    2\n",
       "299957    0\n",
       "299965    4\n",
       "299977    1\n",
       "Name: ord_2, Length: 25204, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "pandas_df[pandas_df['month'] == 12]['ord_2']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.3.2 dask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1.3.1.1 Mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 692 ms, sys: 81 ms, total: 773 ms\n",
      "Wall time: 764 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "nom_1\n",
       "0    5945.734459\n",
       "1    5950.621687\n",
       "2    5976.289594\n",
       "3    5942.845329\n",
       "4    5964.401518\n",
       "5    6000.618657\n",
       "Name: nom_9, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "dask_df.groupby('nom_1').nom_9.mean().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1.3.1.2 Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## not available"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1.3.1.3 find"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hauntedsoul/anaconda3/lib/python3.7/site-packages/dask/dataframe/core.py:3253: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  meta = self._meta[_extract_meta(key)]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 667 ms, sys: 39.4 ms, total: 706 ms\n",
      "Wall time: 704 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "16        0\n",
       "19        0\n",
       "28        2\n",
       "48        4\n",
       "50        2\n",
       "         ..\n",
       "299894    1\n",
       "299954    2\n",
       "299957    0\n",
       "299965    4\n",
       "299977    1\n",
       "Name: ord_2, Length: 25204, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "dask_df[pandas_df['month'] == 12]['ord_2'].compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.1 standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--\n",
      "CPU times: user 74 ms, sys: 6.88 ms, total: 80.9 ms\n",
      "Wall time: 90.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn import preprocessing\n",
    "def standardization(data_frame,coloumn_name):\n",
    "    standard_scaler = preprocessing.StandardScaler()\n",
    "    data_frame[coloumn_name] = standard_scaler.fit_transform(data_frame[coloumn_name])\n",
    "    return data_frame\n",
    "standardization(pandas_df,['month'])\n",
    "print(\"--\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.2 nomalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---\n",
      "CPU times: user 86.5 ms, sys: 19.9 ms, total: 106 ms\n",
      "Wall time: 102 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn import preprocessing\n",
    "def nomalization(data_frame,coloumn_name):\n",
    "    min_max_scaler = preprocessing.MinMaxScaler()\n",
    "    data_frame[coloumn_name] = min_max_scaler.fit_transform(data_frame[coloumn_name])\n",
    "    return data_frame\n",
    "standardization(pandas_df,['month'])\n",
    "print(\"---\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Dask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.1 standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 282 ms, sys: 275 ms, total: 556 ms\n",
      "Wall time: 1.26 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn import preprocessing\n",
    "def standardization(data_frame,coloumn_name):\n",
    "    standard_scaler = preprocessing.StandardScaler()\n",
    "    data_frame[coloumn_name] = standard_scaler.fit_transform(data_frame[coloumn_name])\n",
    "    return data_frame\n",
    "\n",
    "result = dask_df.map_partitions(standardization,['month'])\n",
    "df = result.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.2 nomalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 348 ms, sys: 229 ms, total: 576 ms\n",
      "Wall time: 1.39 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn import preprocessing\n",
    "def nomalization(data_frame,coloumn_name):\n",
    "    min_max_scaler = preprocessing.MinMaxScaler()\n",
    "    data_frame[coloumn_name] = min_max_scaler.fit_transform(data_frame[coloumn_name])\n",
    "    return data_frame\n",
    "result = dask_df.map_partitions(nomalization,['month'])\n",
    "df = result.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1 Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pandas_df.iloc[:,0:24]  \n",
    "y = pandas_df.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 787 ms, sys: 131 ms, total: 917 ms\n",
      "Wall time: 969 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['id', 'target']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "def get_correlation(data, threshold):\n",
    "    corr_col = set()\n",
    "    corrmat = data.corr()\n",
    "    for i in range(len(corrmat.columns)):\n",
    "        for j in range(i):\n",
    "            if abs(corrmat.iloc[i, j])> threshold:\n",
    "                colname = corrmat.columns[i]\n",
    "                corr_col.add(colname)\n",
    "    return list(corr_col)\n",
    "\n",
    "get_correlation(pandas_df,0.057)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 192 ms, sys: 19.1 ms, total: 211 ms\n",
      "Wall time: 312 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['bin_0', 'bin_3', 'bin_2', 'nom_9', 'nom_5', 'nom_2', 'day', 'nom_8',\n",
       "       'nom_1', 'bin_4'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.feature_selection import chi2\n",
    "def chi_square(X,y,n):\n",
    "    chi_scores = chi2(X,y)\n",
    "    p_values = pd.Series(chi_scores[1],index = X.columns)\n",
    "    p_values.sort_values(ascending = False , inplace = True)\n",
    "    return p_values.index[0:n]\n",
    "chi_square(X,y,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.2 Dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dask_df.iloc[:,0:24]  \n",
    "y = dask_df.iloc[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dask.dataframe.core.DataFrame"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.38 s, sys: 629 ms, total: 2.01 s\n",
      "Wall time: 5.19 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['id', 'target']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "def get_correlation(data, threshold):\n",
    "    corr_col = set()\n",
    "    corrmat = data.corr()\n",
    "    for i in range(len(corrmat.columns)):\n",
    "        for j in range(i):\n",
    "            if abs(corrmat.iloc[i, j])> threshold:\n",
    "                colname = corrmat.columns[i]\n",
    "                corr_col.add(colname)\n",
    "    return list(corr_col)\n",
    "dask_df=dask_df.compute()\n",
    "get_correlation(dask_df,0.057)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.04 s, sys: 551 ms, total: 2.6 s\n",
      "Wall time: 8.94 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['bin_0', 'bin_3', 'bin_2', 'nom_9', 'nom_5', 'nom_2', 'day', 'nom_8',\n",
       "       'nom_1', 'bin_4'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.feature_selection import chi2\n",
    "def chi_square(X,y,n):\n",
    "    chi_scores = chi2(X,y)\n",
    "    p_values = pd.Series(chi_scores[1],index = X.columns)\n",
    "    p_values.sort_values(ascending = False , inplace = True)\n",
    "    return p_values.index[0:n]\n",
    "chi_square(X,y,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.Machine learning models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = dask_df['target']\n",
    "X=dask_df.drop(['target','id','Unnamed: 0'], axis=1)\n",
    "y=y.compute()\n",
    "X=X.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 NON Distributed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 Normal GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.1.1 RandomForest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hauntedsoul/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.68 s, sys: 852 ms, total: 7.53 s\n",
      "Wall time: 28.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv='warn', error_score='raise-deprecating',\n",
       "             estimator=RandomForestClassifier(bootstrap=True, class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features='auto',\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=1,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              n_estimators='warn', n_jobs=None,\n",
       "                                              oob_score=False,\n",
       "                                              random_state=None, verbose=0,\n",
       "                                              warm_start=False),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid={'max_depth': [1, 2, 3, 4, 5],\n",
       "                         'n_estimators': [1, 2, 3, 4, 5]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "nEstimator = [1,2,3,4,5]\n",
    "depth = [1,2,3,4,5]\n",
    "RF = RandomForestClassifier()\n",
    "hyperParam = {'n_estimators':nEstimator,'max_depth': depth}\n",
    "grid_search = GridSearchCV(RF,hyperParam,n_jobs=-1)\n",
    "grid_search.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4.1.2 Logistic Regression "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hauntedsoul/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "/home/hauntedsoul/anaconda3/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py:706: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  \"timeout or by a memory leak.\", UserWarning\n",
      "/home/hauntedsoul/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31.8 s, sys: 3.16 s, total: 34.9 s\n",
      "Wall time: 1min 25s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv='warn', error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='warn',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=-1, param_grid={'C': [0.0001, 0.001, 0.01]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "C=[10**-4,10**-3,10**-2]\n",
    "\n",
    "LR = LogisticRegression()\n",
    "hyperParam = {'C':C}\n",
    "grid_search = GridSearchCV(LR,hyperParam,n_jobs=-1)\n",
    "\n",
    "grid_search.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hauntedsoul/anaconda3/lib/python3.7/site-packages/distributed/dashboard/core.py:72: UserWarning: \n",
      "Port 8787 is already in use. \n",
      "Perhaps you already have a cluster running?\n",
      "Hosting the diagnostics dashboard on a random port instead.\n",
      "  warnings.warn(\"\\n\" + msg)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Client</h3>\n",
       "<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
       "  <li><b>Scheduler: </b>tcp://127.0.0.1:38735</li>\n",
       "  <li><b>Dashboard: </b><a href='http://127.0.0.1:37241/status' target='_blank'>http://127.0.0.1:37241/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3 style=\"text-align: left;\">Cluster</h3>\n",
       "<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
       "  <li><b>Workers: </b>4</li>\n",
       "  <li><b>Cores: </b>4</li>\n",
       "  <li><b>Memory: </b>8.25 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: 'tcp://127.0.0.1:38735' processes=4 threads=4, memory=8.25 GB>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dask.distributed import Client, progress\n",
    "import time\n",
    "\n",
    "client = Client()\n",
    "client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.73 s, sys: 1.82 s, total: 11.6 s\n",
      "Wall time: 35.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from dask_ml.model_selection import GridSearchCV\n",
    "import joblib\n",
    "nEstimator = [1,2,3,4,5]\n",
    "depth = [1,2,3,4,5]\n",
    "RF = RandomForestClassifier()\n",
    "hyperParam = [{'n_estimators':nEstimator,'max_depth': depth}]\n",
    "grid_search = GridSearchCV(RF,hyperParam,scoring='f1_weighted')\n",
    "import joblib\n",
    "\n",
    "with joblib.parallel_backend('dask',scatter=[X, y]):\n",
    "    grid_search.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26.7 s, sys: 5.05 s, total: 31.8 s\n",
      "Wall time: 1min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from dask_ml.model_selection import GridSearchCV\n",
    "import joblib\n",
    "C=[10**-4,10**-3,10**-2]\n",
    "\n",
    "LR = LogisticRegression()\n",
    "hyperParam = {'C':C}\n",
    "grid_search = GridSearchCV(LR,hyperParam,n_jobs=-1)\n",
    "import joblib\n",
    "\n",
    "with joblib.parallel_backend('dask',scatter=[X, y]):\n",
    "    grid_search.fit(X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "85"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}