data_handling.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "        0         1         2         3         4         5         6    \\\n0 -1.353538  0.502228  0.200049 -0.385999 -4.830360 -0.466474 -0.114870   \n1  0.330628 -0.008287 -0.151095 -0.452316  1.857310 -0.708909  0.870974   \n2  2.397124  0.253787 -0.562319  0.691857  1.259154 -0.137145  0.836201   \n3 -0.045589  0.013519  1.692190 -0.881866 -0.963860 -0.192448  2.453908   \n4 -0.258368  0.334764 -0.524129  0.547756 -0.619213 -0.574689 -1.027528   \n\n        7         8         9    ...       91        92        93        94   \\\n0 -0.645522  0.286933 -0.030947  ...  0.479033  0.908672 -0.474012  0.744562   \n1 -1.233560  0.529006  1.042035  ...  1.615529 -0.776038 -0.098725  0.869339   \n2  1.002106  0.474555 -0.198557  ...  0.559022 -1.325767 -1.442353  0.684389   \n3  0.580090  0.122833 -0.483986  ... -0.113903  1.453723  0.849300 -1.913192   \n4  0.416190  2.412388 -1.379478  ...  0.070707 -0.132563 -0.925480 -0.485734   \n\n        95        96        97        98        99   100  \n0  1.042413 -0.680929 -0.116795 -0.094668 -0.198689    8  \n1 -1.100121  0.564294  0.028877 -0.153127  0.304131    5  \n2  1.656742  1.150041 -0.183686 -0.045855  0.280040    0  \n3 -1.422723  2.672777  0.181795 -0.309590 -1.319934    6  \n4 -0.279630 -1.052839 -0.720411 -1.370074 -0.209224    4  \n\n[5 rows x 101 columns]\n"
     ]
    }
   ],
   "source": [
    "# normalization\n",
    " \n",
    "df = pd.read_csv('/home/kitten/BA_FIRES/fires/datasets/Multiclass/dataset_1_train.csv',header=None)\n",
    "print(df.head())\n",
    "target = df[100]\n",
    "normalized_df = pd.DataFrame(MinMaxScaler().fit_transform(df.drop(columns=100)))\n",
    "normalized_df[100] = target\n",
    "normalized_df.columns = df.columns\n",
    "normalized_df.to_csv('datasets/Multiclass/dataset_1_train_norm.csv', index=None, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "           0         1         2         3         4         5         6    \\\n",
       "0     0.452204  0.679452  0.523251  0.833042  0.436455  0.505958  0.525821   \n",
       "1     0.643209  0.553250  0.569284  0.528615  0.527016  0.557969  0.487829   \n",
       "2     0.314511  0.347335  0.460609  0.360343  0.513086  0.492635  0.229036   \n",
       "3     0.520566  0.442767  0.467080  0.368179  0.487897  0.467946  0.468201   \n",
       "4     0.677153  0.625735  0.276960  0.745617  0.576981  0.359903  0.453722   \n",
       "...        ...       ...       ...       ...       ...       ...       ...   \n",
       "4194  0.533372  0.162916  0.365661  0.646283  0.525307  0.414082  0.371495   \n",
       "4195  0.632415  0.363245  0.557226  0.333666  0.433296  0.495699  0.561164   \n",
       "4196  0.559921  0.877302  0.395320  0.550712  0.197466  0.365461  0.519478   \n",
       "4197  0.518693  0.513666  0.508553  0.414535  0.551853  0.480533  0.654541   \n",
       "4198  0.484340  0.514944  0.479358  0.258760  0.675216  0.406888  0.453402   \n",
       "\n",
       "           7         8         9    ...       241       242       243  \\\n",
       "0     0.455937  0.489096  0.452043  ...  0.373920  0.423160  0.722618   \n",
       "1     0.492363  0.160387  0.491291  ...  0.550554  0.618980  0.582243   \n",
       "2     0.505619  0.578972  0.549618  ...  0.426836  0.394882  0.658356   \n",
       "3     0.494753  0.489169  0.449764  ...  0.593662  0.611698  0.468062   \n",
       "4     0.508704  0.325711  0.560475  ...  0.441665  0.593364  0.505651   \n",
       "...        ...       ...       ...  ...       ...       ...       ...   \n",
       "4194  0.651029  0.183024  0.497612  ...  0.320009  0.517632  0.425821   \n",
       "4195  0.681412  0.386094  0.417753  ...  0.588600  0.652233  0.215870   \n",
       "4196  0.503455  0.382671  0.489422  ...  0.741142  0.472118  0.735945   \n",
       "4197  0.634260  0.229060  0.547271  ...  0.204382  0.518369  0.496953   \n",
       "4198  0.448395  0.455596  0.532427  ...  0.406663  0.520047  0.451682   \n",
       "\n",
       "           244       245       246       247       248       249  250  \n",
       "0     0.602215  0.469015  0.291525  0.447546  0.513351  0.556063    7  \n",
       "1     0.373469  0.663250  0.361567  0.365592  0.703123  0.345732    2  \n",
       "2     0.552663  0.447423  0.370936  0.474540  0.496491  0.576561    7  \n",
       "3     0.450250  0.498112  0.673788  0.297568  0.478575  0.262945    3  \n",
       "4     0.638373  0.565950  0.485470  0.700931  0.124574  0.523484    3  \n",
       "...        ...       ...       ...       ...       ...       ...  ...  \n",
       "4194  0.330396  0.564041  0.478298  0.482125  0.781692  0.531278    2  \n",
       "4195  0.425448  0.648620  0.615221  0.234269  0.627417  0.371394    2  \n",
       "4196  0.508166  0.398665  0.346237  0.412194  0.658402  0.502954    7  \n",
       "4197  0.440677  0.310342  0.409058  0.344250  0.642575  0.527691    2  \n",
       "4198  0.442181  0.319098  0.305198  0.292561  0.533642  0.505054    7  \n",
       "\n",
       "[4199 rows x 251 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>...</th>\n      <th>241</th>\n      <th>242</th>\n      <th>243</th>\n      <th>244</th>\n      <th>245</th>\n      <th>246</th>\n      <th>247</th>\n      <th>248</th>\n      <th>249</th>\n      <th>250</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.452204</td>\n      <td>0.679452</td>\n      <td>0.523251</td>\n      <td>0.833042</td>\n      <td>0.436455</td>\n      <td>0.505958</td>\n      <td>0.525821</td>\n      <td>0.455937</td>\n      <td>0.489096</td>\n      <td>0.452043</td>\n      <td>...</td>\n      <td>0.373920</td>\n      <td>0.423160</td>\n      <td>0.722618</td>\n      <td>0.602215</td>\n      <td>0.469015</td>\n      <td>0.291525</td>\n      <td>0.447546</td>\n      <td>0.513351</td>\n      <td>0.556063</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.643209</td>\n      <td>0.553250</td>\n      <td>0.569284</td>\n      <td>0.528615</td>\n      <td>0.527016</td>\n      <td>0.557969</td>\n      <td>0.487829</td>\n      <td>0.492363</td>\n      <td>0.160387</td>\n      <td>0.491291</td>\n      <td>...</td>\n      <td>0.550554</td>\n      <td>0.618980</td>\n      <td>0.582243</td>\n      <td>0.373469</td>\n      <td>0.663250</td>\n      <td>0.361567</td>\n      <td>0.365592</td>\n      <td>0.703123</td>\n      <td>0.345732</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.314511</td>\n      <td>0.347335</td>\n      <td>0.460609</td>\n      <td>0.360343</td>\n      <td>0.513086</td>\n      <td>0.492635</td>\n      <td>0.229036</td>\n      <td>0.505619</td>\n      <td>0.578972</td>\n      <td>0.549618</td>\n      <td>...</td>\n      <td>0.426836</td>\n      <td>0.394882</td>\n      <td>0.658356</td>\n      <td>0.552663</td>\n      <td>0.447423</td>\n      <td>0.370936</td>\n      <td>0.474540</td>\n      <td>0.496491</td>\n      <td>0.576561</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.520566</td>\n      <td>0.442767</td>\n      <td>0.467080</td>\n      <td>0.368179</td>\n      <td>0.487897</td>\n      <td>0.467946</td>\n      <td>0.468201</td>\n      <td>0.494753</td>\n      <td>0.489169</td>\n      <td>0.449764</td>\n      <td>...</td>\n      <td>0.593662</td>\n      <td>0.611698</td>\n      <td>0.468062</td>\n      <td>0.450250</td>\n      <td>0.498112</td>\n      <td>0.673788</td>\n      <td>0.297568</td>\n      <td>0.478575</td>\n      <td>0.262945</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.677153</td>\n      <td>0.625735</td>\n      <td>0.276960</td>\n      <td>0.745617</td>\n      <td>0.576981</td>\n      <td>0.359903</td>\n      <td>0.453722</td>\n      <td>0.508704</td>\n      <td>0.325711</td>\n      <td>0.560475</td>\n      <td>...</td>\n      <td>0.441665</td>\n      <td>0.593364</td>\n      <td>0.505651</td>\n      <td>0.638373</td>\n      <td>0.565950</td>\n      <td>0.485470</td>\n      <td>0.700931</td>\n      <td>0.124574</td>\n      <td>0.523484</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4194</th>\n      <td>0.533372</td>\n      <td>0.162916</td>\n      <td>0.365661</td>\n      <td>0.646283</td>\n      <td>0.525307</td>\n      <td>0.414082</td>\n      <td>0.371495</td>\n      <td>0.651029</td>\n      <td>0.183024</td>\n      <td>0.497612</td>\n      <td>...</td>\n      <td>0.320009</td>\n      <td>0.517632</td>\n      <td>0.425821</td>\n      <td>0.330396</td>\n      <td>0.564041</td>\n      <td>0.478298</td>\n      <td>0.482125</td>\n      <td>0.781692</td>\n      <td>0.531278</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>4195</th>\n      <td>0.632415</td>\n      <td>0.363245</td>\n      <td>0.557226</td>\n      <td>0.333666</td>\n      <td>0.433296</td>\n      <td>0.495699</td>\n      <td>0.561164</td>\n      <td>0.681412</td>\n      <td>0.386094</td>\n      <td>0.417753</td>\n      <td>...</td>\n      <td>0.588600</td>\n      <td>0.652233</td>\n      <td>0.215870</td>\n      <td>0.425448</td>\n      <td>0.648620</td>\n      <td>0.615221</td>\n      <td>0.234269</td>\n      <td>0.627417</td>\n      <td>0.371394</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>4196</th>\n      <td>0.559921</td>\n      <td>0.877302</td>\n      <td>0.395320</td>\n      <td>0.550712</td>\n      <td>0.197466</td>\n      <td>0.365461</td>\n      <td>0.519478</td>\n      <td>0.503455</td>\n      <td>0.382671</td>\n      <td>0.489422</td>\n      <td>...</td>\n      <td>0.741142</td>\n      <td>0.472118</td>\n      <td>0.735945</td>\n      <td>0.508166</td>\n      <td>0.398665</td>\n      <td>0.346237</td>\n      <td>0.412194</td>\n      <td>0.658402</td>\n      <td>0.502954</td>\n      <td>7</td>\n    </tr>\n    <tr>\n      <th>4197</th>\n      <td>0.518693</td>\n      <td>0.513666</td>\n      <td>0.508553</td>\n      <td>0.414535</td>\n      <td>0.551853</td>\n      <td>0.480533</td>\n      <td>0.654541</td>\n      <td>0.634260</td>\n      <td>0.229060</td>\n      <td>0.547271</td>\n      <td>...</td>\n      <td>0.204382</td>\n      <td>0.518369</td>\n      <td>0.496953</td>\n      <td>0.440677</td>\n      <td>0.310342</td>\n      <td>0.409058</td>\n      <td>0.344250</td>\n      <td>0.642575</td>\n      <td>0.527691</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>4198</th>\n      <td>0.484340</td>\n      <td>0.514944</td>\n      <td>0.479358</td>\n      <td>0.258760</td>\n      <td>0.675216</td>\n      <td>0.406888</td>\n      <td>0.453402</td>\n      <td>0.448395</td>\n      <td>0.455596</td>\n      <td>0.532427</td>\n      <td>...</td>\n      <td>0.406663</td>\n      <td>0.520047</td>\n      <td>0.451682</td>\n      <td>0.442181</td>\n      <td>0.319098</td>\n      <td>0.305198</td>\n      <td>0.292561</td>\n      <td>0.533642</td>\n      <td>0.505054</td>\n      <td>7</td>\n    </tr>\n  </tbody>\n</table>\n<p>4199 rows × 251 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "normalized_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle data and split into test and train\n",
    "df = pd.read_csv('datasets/Multiclass/covtype.scale01.csv')\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df[:400000]\n",
    "df_test = df[400001:]\n",
    "df_test.to_csv('datasets/Multiclass/covtype.scale01.test.csv', index=False)\n",
    "df_train.to_csv('datasets/Multiclass/covtype.scale01.train.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('datasets/Multiclass/har_train.csv', header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[561]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}