8_best_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_df = pd.read_csv('data/dataset/oversampling/dfu_features_dataset.csv', index_col=0)\n",
    "y_df = pd.read_csv('data/dataset/oversampling/dfu_labels_dataset.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_parameters(features_df: pd.DataFrame, n_features:int) -> pd.DataFrame:\n",
    "    features = features_df.iloc[:n_features]['Features'].values\n",
    "    X = X_df[features].to_numpy().astype(np.float32)\n",
    "    y = y_df.to_numpy().ravel()\n",
    "\n",
    "    param_grid = {'C': np.logspace(0, 2, 25), 'gamma': np.logspace(-3, 0, 25), 'degree':np.arange(3, 8, 1),\n",
    "         'kernel': ['rbf', 'linear', 'poly'], 'probability': [True], 'random_state': [42]}\n",
    "\n",
    "    grid = RandomizedSearchCV(SVC(), param_grid, refit=True, verbose=2, cv=5, n_jobs=-1, n_iter=500, random_state=42)\n",
    "    grid.fit(X, y)\n",
    "\n",
    "    return pd.DataFrame(grid.best_params_, index=[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_df = pd.read_csv('data/features_importance/oversampling/lasso.csv')\n",
    "n_parameters = [10, 25, 50]\n",
    "\n",
    "for i in n_parameters:\n",
    "    df = find_parameters(features_df, i)\n",
    "    # save best params to csv file\n",
    "    df.to_csv('data/best_params/svm_lasso_{}.csv'.format(i), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Concrete Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_df = pd.read_csv('data/features_importance/oversampling/concrete_dropout.csv')\n",
    "\n",
    "n_parameters = [10, 25, 50]\n",
    "\n",
    "for i in n_parameters:\n",
    "    df = find_parameters(features_df, i)\n",
    "    # save best params to csv file\n",
    "    df.to_csv('data/best_params/svm_concrete_{}.csv'.format(i), index=False)\n",
    "\n",
    "n_parameters = [10, 25, 50]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# All features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X_df.to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "param_grid = {'C': np.logspace(0, 2, 10), 'gamma': np.logspace(-3, 0, 12), 'degree':[3,4,5,6], 'kernel': ['rbf', 'linear', 'poly'],\n",
    "        'probability': [True], 'random_state': [42]}\n",
    "\n",
    "grid = RandomizedSearchCV(SVC(), param_grid, refit=True, verbose=2, cv=5, n_jobs=-1, n_iter=100, random_state=42)\n",
    "grid.fit(X, y)\n",
    "\n",
    "df = pd.DataFrame(grid.best_params_, index=[0])\n",
    "df.to_csv('data/best_params/svm_all.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_results_save_dir = 'data/best_params/results/'\n",
    "\n",
    "columns = pd.MultiIndex.from_tuples([('Accuracy', 'Mean'), ('Accuracy', 'Std'), \n",
    "                                     ('Precision', 'Mean'), ('Precision', 'Std'), \n",
    "                                     ('Recall', 'Mean'), ('Recall', 'Std'),\n",
    "                                      ('F1-Score', 'Mean'), ('F1-Score', 'Std')])\n",
    "\n",
    "metrics_df = pd.DataFrame(columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(X, y, **svc_params):\n",
    "    from sklearn.svm import SVC\n",
    "    from sklearn.model_selection import KFold\n",
    "\n",
    "    k_folds = 5\n",
    "    kfold = KFold(n_splits=k_folds, shuffle=True, random_state=42)\n",
    "\n",
    "    y_true = []\n",
    "    y_predict = [] \n",
    "\n",
    "    for fold, (train_ids, test_ids) in enumerate(kfold.split(X)):\n",
    "        X_train, X_test = X[train_ids], X[test_ids]\n",
    "        y_train, y_test = y[train_ids], y[test_ids]\n",
    "        svc = SVC(**svc_params)\n",
    "        svc.fit(X_train, y_train)\n",
    "\n",
    "        y_true.append(y_test)\n",
    "        y_predict.append(svc.predict(X_test))\n",
    "\n",
    "        print(f'Acc. Fold {fold}: {svc.score(X_test, y_test)}')\n",
    "\n",
    "    return y_true, y_predict\n",
    "\n",
    "def compute_metrics(y_true: list, y_predict: list):\n",
    "    from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "\n",
    "    acc, prec, rec, f1 = np.zeros(len(y_true)), np.zeros(len(y_true)), np.zeros(len(y_true)), np.zeros(len(y_true))\n",
    "\n",
    "    for cv_idx in range(len(y_true)):\n",
    "        acc[cv_idx] = accuracy_score(y_true[cv_idx], y_predict[cv_idx])\n",
    "        prec[cv_idx] = precision_score(y_true[cv_idx], y_predict[cv_idx]) \n",
    "        rec[cv_idx] = recall_score(y_true[cv_idx], y_predict[cv_idx])\n",
    "        f1[cv_idx] = f1_score(y_true[cv_idx], y_predict[cv_idx])\n",
    "\n",
    "    return acc, prec, rec, f1\n",
    "\n",
    "def plot_confusion_matrix(y_true, y_predict):\n",
    "    from sklearn.metrics import ConfusionMatrixDisplay\n",
    "    from matplotlib import pyplot as plt\n",
    "    labels = ['CG', 'DM']\n",
    "    cmp = ConfusionMatrixDisplay.from_predictions(y_true, y_predict, display_labels=labels, cmap='Blues', normalize='true')\n",
    "    fig, ax = plt.subplots(figsize=(10,8))\n",
    "    cmp.plot(ax=ax, cmap='Blues')\n",
    "    # Increase labels size\n",
    "    ax.tick_params(axis='both', which='major', labelsize=20)\n",
    "    # Increase xlabel size\n",
    "    ax.set_xlabel(ax.get_xlabel(), fontsize=18)\n",
    "    ax.set_ylabel(ax.get_ylabel(), fontsize=20)\n",
    "    #Increase values size\n",
    "    for text in ax.texts:\n",
    "        text.set_size(18)\n",
    "    # Increase colorbar ticks size\n",
    "    ax.figure.axes[-1].tick_params(labelsize=18)\n",
    "    \n",
    "    return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## All features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.9183673469387755\n",
      "Acc. Fold 1: 0.9795918367346939\n",
      "Acc. Fold 2: 0.9387755102040817\n",
      "Acc. Fold 3: 0.7959183673469388\n",
      "Acc. Fold 4: 0.9166666666666666\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_all.csv').to_dict(orient='records')[0]\n",
    "X = X_df.to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['All Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/All_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LASSO\n",
    "### 10 First Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.9387755102040817\n",
      "Acc. Fold 1: 0.9183673469387755\n",
      "Acc. Fold 2: 0.8979591836734694\n",
      "Acc. Fold 3: 0.8367346938775511\n",
      "Acc. Fold 4: 0.8958333333333334\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_lasso_10.csv').to_dict(orient='records')[0]\n",
    "features_df = pd.read_csv('data/features_importance/oversampling/lasso.csv')\n",
    "features = features_df.iloc[:10]['Features'].values\n",
    "\n",
    "X = X_df[features].to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)\n",
    "\n",
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['LASSO 10 Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/LASSO_10_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 25 First Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.9591836734693877\n",
      "Acc. Fold 1: 0.9591836734693877\n",
      "Acc. Fold 2: 0.9387755102040817\n",
      "Acc. Fold 3: 0.9183673469387755\n",
      "Acc. Fold 4: 0.9583333333333334\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_lasso_25.csv').to_dict(orient='records')[0]\n",
    "features_df = pd.read_csv('data/features_importance/oversampling/lasso.csv')\n",
    "features = features_df.iloc[:25]['Features'].values\n",
    "\n",
    "X = X_df[features].to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)\n",
    "\n",
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['LASSO 25 Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/LASSO_25_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 50 First Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.9795918367346939\n",
      "Acc. Fold 1: 0.9795918367346939\n",
      "Acc. Fold 2: 0.9183673469387755\n",
      "Acc. Fold 3: 0.7755102040816326\n",
      "Acc. Fold 4: 0.9166666666666666\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_lasso_50.csv').to_dict(orient='records')[0]\n",
    "features_df = pd.read_csv('data/features_importance/oversampling/lasso.csv')\n",
    "features = features_df.iloc[:50]['Features'].values\n",
    "\n",
    "X = X_df[features].to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)\n",
    "\n",
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['LASSO 50 Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/LASSO_50_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concrete Dropout\n",
    "### 10 First Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.8979591836734694\n",
      "Acc. Fold 1: 0.9387755102040817\n",
      "Acc. Fold 2: 0.9183673469387755\n",
      "Acc. Fold 3: 0.7755102040816326\n",
      "Acc. Fold 4: 0.9166666666666666\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_concrete_10.csv').to_dict(orient='records')[0]\n",
    "features_df = pd.read_csv('data/features_importance/oversampling/concrete_dropout.csv')\n",
    "features = features_df.iloc[:10]['Features'].values\n",
    "\n",
    "X = X_df[features].to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)\n",
    "\n",
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['Concrete 10 Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/Concrete_10_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 25 First Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.8979591836734694\n",
      "Acc. Fold 1: 1.0\n",
      "Acc. Fold 2: 0.8979591836734694\n",
      "Acc. Fold 3: 0.7551020408163265\n",
      "Acc. Fold 4: 0.9166666666666666\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_concrete_25.csv').to_dict(orient='records')[0]\n",
    "features_df = pd.read_csv('data/features_importance/oversampling/concrete_dropout.csv')\n",
    "features = features_df.iloc[:25]['Features'].values\n",
    "\n",
    "X = X_df[features].to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)\n",
    "\n",
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['Concrete 25 Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/Concrete_25_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 50 First Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc. Fold 0: 0.8979591836734694\n",
      "Acc. Fold 1: 0.9591836734693877\n",
      "Acc. Fold 2: 0.8571428571428571\n",
      "Acc. Fold 3: 0.7959183673469388\n",
      "Acc. Fold 4: 0.875\n"
     ]
    }
   ],
   "source": [
    "param = pd.read_csv('data/best_params/svm_concrete_50.csv').to_dict(orient='records')[0]\n",
    "features_df = pd.read_csv('data/features_importance/oversampling/concrete_dropout.csv')\n",
    "features = features_df.iloc[:50]['Features'].values\n",
    "\n",
    "X = X_df[features].to_numpy().astype(np.float32)\n",
    "y = y_df.to_numpy().ravel()\n",
    "\n",
    "y_true, y_predict = test(X, y, **param)\n",
    "\n",
    "acc, prec, rec, f1 = compute_metrics(y_true, y_predict)\n",
    "metrics_df.loc['Concrete 50 Features'] = [acc.mean(), acc.std(), prec.mean(), prec.std(), rec.mean(), rec.std(), f1.mean(), f1.std()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure = plot_confusion_matrix(y_true, y_predict)\n",
    "figure.savefig(os.path.join(testing_results_save_dir, 'imgs/Concrete_50_Features.pdf'), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(os.path.join(testing_results_save_dir, 'metrics.csv'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('DeepLearning')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "1d9b8aa8d774518be7ebcfd06a2463a8035a66798fac49b1a363f570d2d8622e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}