no_norm_problem.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python39164bit782dbb36b7b848238b8e7e5b770c6244",
   "display_name": "Python 3.9.1 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the functions needed for validate and comparsion\n",
    "\n",
    "import numpy as np\n",
    "import cupy as cp\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "#from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from skmultiflow.data import FileStream\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "\n",
    "# using plotly for plots\n",
    "#import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "from plotly.subplots import make_subplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fires_prints import FIRES\n",
    "from stability import stability_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Human Activity Recognition\n",
    "# labels changed from [1,...,6] to [0,...,5]\n",
    "# rows shuffled\n",
    "# split into train set with 7352 instances and test set with 2948\n",
    "stream = FileStream('datasets/Multiclass/har_train.csv', target_idx = 561)\n",
    "dataset_name = \"har\"\n",
    "n_selected_ftr = 100\n",
    "n_window = 10\n",
    "batch_size = 50\n",
    "weights = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "MLPClassifier()"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "stream.restart()\n",
    "predictor = MLPClassifier()\n",
    "x,y = stream.next_sample(batch_size=batch_size)\n",
    "predictor.partial_fit(x,y, stream.target_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "fires_model = FIRES(n_total_ftr=stream.n_features,\n",
    "                    target_values=stream.target_values,\n",
    "                    mu_init=0,\n",
    "                    sigma_init=1,\n",
    "                    model='softmax',\n",
    "                    class_probabilities=weights,\n",
    "                    lr_mu=0.2,\n",
    "                    lr_sigma=0.2,\n",
    "                    number_monte_carlo_samples=100)               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "fires_cuda_accuracy = []\n",
    "\n",
    "fires_cuda_selected_ftrs = []\n",
    "fires_cuda_stability = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "one call\n",
      "50\n",
      "For class 0 there are 7 obs\n",
      "r shape(7, 100, 561, 6)\n",
      "4.99406718420424\n",
      "-5.148857807929787\n",
      "theta:\n",
      "4.99464886922113\n",
      "-5.146656279449741\n",
      "theta times x\n",
      "(7, 100, 6)\n",
      "52.637543080593105\n",
      "-48.675118611049314\n",
      "-6.77259777908707\n",
      "mean:\n",
      "(7, 100)\n",
      "18.144282156674958\n",
      "-17.217085223567775\n",
      "median:\n",
      "(7, 100)\n",
      "19.900315934872637\n",
      "-18.452921721448426\n",
      "a shape (7, 100)\n",
      "6.77259777908707\n",
      "-52.637543080593105\n",
      "eta:\n",
      "(7, 100, 6)\n",
      "False\n",
      "0.0\n",
      "-95.58881487186002\n",
      "exp eta:\n",
      "1.0\n",
      "3.064115766270318e-42\n",
      "eta_sum\n",
      "(7, 100)\n",
      "2.546123026708787\n",
      "1.0\n",
      "softmax_all\n",
      "(7, 100, 6)\n",
      "1.0\n",
      "3.0641157662703176e-42\n",
      "False\n",
      "(7, 100)\n",
      "marginal\n",
      "(7,)\n",
      "0.23033944789180325\n",
      "0.11189535486728801\n",
      "False\n",
      "softmax_derivative\n",
      "(7, 100, 561, 6)\n",
      "0.24864123768479954\n",
      "-0.24905290534495508\n",
      "False\n",
      "0.005339267259457211\n",
      "1.0016874888845615\n",
      "For class 1 there are 4 obs\n",
      "r shape(4, 100, 561, 6)\n",
      "5.250338279504917\n",
      "-4.545697888686622\n",
      "theta:\n",
      "5.249874314909863\n",
      "-4.5452318498091415\n",
      "theta times x\n",
      "(4, 100, 6)\n",
      "50.790665919955245\n",
      "-51.8103122416785\n",
      "-2.9393159398170834\n",
      "mean:\n",
      "(4, 100)\n",
      "18.486980846114957\n",
      "-19.565582608608924\n",
      "median:\n",
      "(4, 100)\n",
      "21.462286681706466\n",
      "-19.808273153182242\n",
      "a shape (4, 100)\n",
      "2.9393159398170834\n",
      "-50.790665919955245\n",
      "eta:\n",
      "(4, 100, 6)\n",
      "False\n",
      "0.0\n",
      "-93.24536528515071\n",
      "exp eta:\n",
      "1.0\n",
      "3.191922907803145e-41\n",
      "eta_sum\n",
      "(4, 100)\n",
      "2.675884882493922\n",
      "1.0\n",
      "softmax_all\n",
      "(4, 100, 6)\n",
      "1.0\n",
      "3.1919229078031446e-41\n",
      "False\n",
      "(4, 100)\n",
      "marginal\n",
      "(4,)\n",
      "0.21391255327831765\n",
      "0.12194748642291461\n",
      "False\n",
      "softmax_derivative\n",
      "(4, 100, 561, 6)\n",
      "0.24265214606038346\n",
      "-0.24286599132384643\n",
      "False\n",
      "0.005171428586590842\n",
      "1.0016168495412405\n",
      "For class 2 there are 6 obs\n",
      "r shape(6, 100, 561, 6)\n",
      "4.976814942593735\n",
      "-4.790689598615385\n",
      "theta:\n",
      "4.975984841893243\n",
      "-4.790794594039958\n",
      "theta times x\n",
      "(6, 100, 6)\n",
      "52.12646407473533\n",
      "-49.13662521986928\n",
      "-5.730474958475054\n",
      "mean:\n",
      "(6, 100)\n",
      "16.19638427216305\n",
      "-15.234446981459191\n",
      "median:\n",
      "(6, 100)\n",
      "16.68593698092598\n",
      "-20.37704949234319\n",
      "a shape (6, 100)\n",
      "5.730474958475054\n",
      "-52.12646407473533\n",
      "eta:\n",
      "(6, 100, 6)\n",
      "False\n",
      "0.0\n",
      "-101.26308929460461\n",
      "exp eta:\n",
      "1.0\n",
      "1.0519596910966767e-44\n",
      "eta_sum\n",
      "(6, 100)\n",
      "3.1468588746081805\n",
      "1.0\n",
      "softmax_all\n",
      "(6, 100, 6)\n",
      "1.0\n",
      "1.0519596909575211e-44\n",
      "False\n",
      "(6, 100)\n",
      "marginal\n",
      "(6,)\n",
      "0.21177529190628608\n",
      "0.14151943001900988\n",
      "False\n",
      "softmax_derivative\n",
      "(6, 100, 561, 6)\n",
      "0.24762201696113836\n",
      "-0.2486673925054803\n",
      "False\n",
      "0.004765295441841681\n",
      "1.0015203993784105\n",
      "For class 3 there are 9 obs\n",
      "r shape(9, 100, 561, 6)\n",
      "4.895543857109659\n",
      "-5.275035966548157\n",
      "theta:\n",
      "4.899445696093527\n",
      "-5.2740244834295575\n",
      "theta times x\n",
      "(9, 100, 6)\n",
      "68.36519004960334\n",
      "-66.56833330121431\n",
      "-10.911142278017511\n",
      "mean:\n",
      "(9, 100)\n",
      "28.470952249236706\n",
      "-22.560092207597336\n",
      "median:\n",
      "(9, 100)\n",
      "32.80280268401911\n",
      "-30.93403812184354\n",
      "a shape (9, 100)\n",
      "10.911142278017511\n",
      "-68.36519004960334\n",
      "eta:\n",
      "(9, 100, 6)\n",
      "False\n",
      "0.0\n",
      "-110.44487998245206\n",
      "exp eta:\n",
      "1.0\n",
      "1.0824256352023952e-48\n",
      "eta_sum\n",
      "(9, 100)\n",
      "2.773553105903036\n",
      "1.0\n",
      "softmax_all\n",
      "(9, 100, 6)\n",
      "1.0\n",
      "1.082425594618408e-48\n",
      "False\n",
      "(9, 100)\n",
      "marginal\n",
      "(9,)\n",
      "0.21865006271392132\n",
      "0.12890909353814958\n",
      "False\n",
      "softmax_derivative\n",
      "(9, 100, 561, 6)\n",
      "0.24998981345559326\n",
      "-0.24999918478970962\n",
      "False\n",
      "0.005066027204318542\n",
      "1.0015839535832427\n",
      "For class 4 there are 10 obs\n",
      "r shape(10, 100, 561, 6)\n",
      "4.820222960418646\n",
      "-4.930469929271645\n",
      "theta:\n",
      "4.814910128355492\n",
      "-4.931517578101165\n",
      "theta times x\n",
      "(10, 100, 6)\n",
      "67.11094313994163\n",
      "-71.22411325887478\n",
      "-8.651637142279416\n",
      "mean:\n",
      "(10, 100)\n",
      "23.57321545501475\n",
      "-23.16413226155603\n",
      "median:\n",
      "(10, 100)\n",
      "27.46169265472679\n",
      "-32.000750896071494\n",
      "a shape (10, 100)\n",
      "8.651637142279416\n",
      "-67.11094313994163\n",
      "eta:\n",
      "(10, 100, 6)\n",
      "False\n",
      "0.0\n",
      "-113.17671297474222\n",
      "exp eta:\n",
      "1.0\n",
      "7.046574942114937e-50\n",
      "eta_sum\n",
      "(10, 100)\n",
      "2.3936824436607296\n",
      "1.0\n",
      "softmax_all\n",
      "(10, 100, 6)\n",
      "1.0\n",
      "7.046574942114937e-50\n",
      "False\n",
      "(10, 100)\n",
      "marginal\n",
      "(10,)\n",
      "0.2194716329756236\n",
      "0.11649411406195882\n",
      "False\n",
      "softmax_derivative\n",
      "(10, 100, 561, 6)\n",
      "0.24985716674049208\n",
      "-0.24998394148592032\n",
      "False\n",
      "0.005323003976433885\n",
      "1.0016366633487235\n",
      "For class 5 there are 14 obs\n",
      "r shape(14, 100, 561, 6)\n",
      "5.246585647811845\n",
      "-4.910995691605172\n",
      "theta:\n",
      "5.2452285552948466\n",
      "-4.912334414354927\n",
      "theta times x\n",
      "(14, 100, 6)\n",
      "67.09473426478405\n",
      "-79.29441999343251\n",
      "-12.194470774768863\n",
      "mean:\n",
      "(14, 100)\n",
      "25.346444474567175\n",
      "-26.83730092653124\n",
      "median:\n",
      "(14, 100)\n",
      "28.972633643637618\n",
      "-32.159605416811196\n",
      "a shape (14, 100)\n",
      "12.194470774768863\n",
      "-67.09473426478405\n",
      "eta:\n",
      "(14, 100, 6)\n",
      "False\n",
      "0.0\n",
      "-115.37790019149716\n",
      "exp eta:\n",
      "1.0\n",
      "7.79856350882058e-51\n",
      "eta_sum\n",
      "(14, 100)\n",
      "2.379845482786875\n",
      "1.0\n",
      "softmax_all\n",
      "(14, 100, 6)\n",
      "1.0\n",
      "7.79856350882058e-51\n",
      "False\n",
      "(14, 100)\n",
      "marginal\n",
      "(14,)\n",
      "0.22346717851342177\n",
      "0.12259935013553636\n",
      "False\n",
      "softmax_derivative\n",
      "(14, 100, 561, 6)\n",
      "0.24980709912333285\n",
      "-0.2498079318601704\n",
      "False\n",
      "0.006876908701688441\n",
      "1.0018022291906188\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "MLPClassifier()"
      ]
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "#for i in range(1):\n",
    "# Load a new sample\n",
    "x, y = stream.next_sample(batch_size=batch_size)\n",
    "# Select features\n",
    "ftr_weights = fires_model.weigh_features(x,y)\n",
    "ftr_selection = np.argsort(ftr_weights)[::-1][:n_selected_ftr]\n",
    "if np.isnan(ftr_weights).any():\n",
    "    print(\"NAN occured\")\n",
    "\n",
    "\n",
    "# Truncate x (retain only selected features, 'remove' all others, e.g. by replacing them with 0)\n",
    "x_reduced = np.zeros(x.shape)\n",
    "x_reduced[:, ftr_selection] = x[:, ftr_selection]\n",
    "\n",
    "# stability test\n",
    "ftr_array = np.zeros(stream.n_features)\n",
    "ftr_array[ftr_selection] = 1\n",
    "fires_cuda_selected_ftrs.append(ftr_array)\n",
    "\n",
    "if len(fires_cuda_selected_ftrs) >= 10:\n",
    "    stability = stability_factor(fires_cuda_selected_ftrs[-10:])\n",
    "    fires_cuda_stability.append(stability)\n",
    "\n",
    "\n",
    "# Test\n",
    "y_pred = predictor.predict(x_reduced)\n",
    "    \n",
    "fires_cuda_accuracy.append(accuracy_score(y, y_pred))\n",
    "\n",
    "\n",
    "# Train\n",
    "predictor.partial_fit(x_reduced, y)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ftr_weights\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.exp(-750)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fires_moving_average = pd.Series(fires_cuda_accuracy).rolling(window=n_window).mean().iloc[n_window-1:].value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = (\"iaeiae {} iaeiae \"\n",
    "        \"iaeiae {} iaeia \"\n",
    "        \"iaeiae\").format(2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'iaeiae 2 iaeiae iaeiae 3 iaeia iaeiae'"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "msg = (\"The marginal likelihood of class {} \"\n",
    "                                   \"reached 0. This will cause the feature \"\n",
    "                                   \"weights to become NaN. Try to normalize \"\n",
    "                                   \"your data or reduce learning rate.\"\n",
    "                                   ).format(\"iaeiae\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'The marginal likelihood of class iaeiae reached 0. This will cause the feature weights to become NaN. Try to normalize your data or reduce learning rate.'"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "msg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "0 in np.array((np.exp(-500),0.3, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}