2_correlation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_df = pd.read_csv('data/dataset/dfu_features_dataset.csv', index_col=0)\n",
    "y_df = pd.read_csv('data/dataset/dfu_labels_dataset.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# matplolib for showing the correlation between variables\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Set up the matplotlib figure\n",
    "f, ax = plt.subplots(figsize=(11, 11))\n",
    "\n",
    "# Generate a custom diverging colormap\n",
    "cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "\n",
    "# Draw the heatmap with the mask and correct aspect ratio\n",
    "sns.heatmap(X_df.corr(), cmap=cmap, vmax=1., vmin=-1, center=0,\n",
    "            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_df[y_df==0].index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation between Control Group and Diabetic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter dataframe by index prefix\n",
    "# cg_group = X_df.filter(regex='^CG', axis=0).to_numpy()\n",
    "# dm_group = X_df.filter(regex='^DM', axis=0).to_numpy()\n",
    "\n",
    "cg_group_idx = y_df[y_df.Label==0].index\n",
    "cg_group = X_df.loc[cg_group_idx]\n",
    "\n",
    "df_group_idx = y_df[y_df.Label==1].index\n",
    "dm_group = X_df.loc[df_group_idx]\n",
    "\n",
    "# calculate the correlation between two groups\n",
    "import numpy as np\n",
    "corr = np.corrcoef(cg_group, dm_group)\n",
    "\n",
    "# Plot correlation matrix\n",
    "plt.figure(figsize=(12, 10))\n",
    "plt.imshow(corr, cmap='Purples', vmin=.5, vmax=1, interpolation='nearest')\n",
    "plt.xticks([])\n",
    "plt.yticks([])\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute ROC-AUC score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.concatenate((cg_group, dm_group), axis=0)\n",
    "y = np.concatenate((np.zeros(cg_group.shape[0]), np.ones(dm_group.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import numpy as np\n",
    "\n",
    "roc_auc = np.zeros(X.shape[1])\n",
    "\n",
    "for feature in range(X.shape[1]):\n",
    "    _X = np.expand_dims(X[:,feature], axis=1)\n",
    "    clf = LogisticRegression(solver=\"liblinear\", random_state=0).fit(_X, y)\n",
    "    y_hat = np.argmax(clf.predict_proba(_X), axis=1)\n",
    "    roc_auc[feature] = roc_auc_score(y, y_hat)\n",
    "\n",
    "# Force AUC >= .5\n",
    "roc_auc = (np.abs(roc_auc - 0.5) + 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt \n",
    "\n",
    "plt.plot(roc_auc)\n",
    "plt.xlabel('Feature')\n",
    "plt.ylabel('AUC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove high-correlated features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sorted by Feature Ranking based on ROC-AUC\n",
    "X_df = X_df.iloc[:,roc_auc.argsort()[::-1]]\n",
    "X_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def select_features(df, threshold=.95):\n",
    "    '''\n",
    "        Select the features and remove those which have a high correlation factor.\n",
    "\n",
    "        The features are selected taking into account the order of the features, i.e.,\n",
    "        if three features are highly-correlated, the first one is selected and the\n",
    "        other two are removed. For this reason, it is important the order of the features\n",
    "        is sorted by the feature ranking such as, for instance, ROC-AUC.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        df : pandas.DataFrame\n",
    "            Dataframe which contains the dataset\n",
    "\n",
    "        threshold : float\n",
    "            Threshold for the correlation factor\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        selected_features : list\n",
    "            List of selected features\n",
    "    '''\n",
    "\n",
    "    tmp_df = df.copy()\n",
    "    mask = np.ones(len(tmp_df.columns), dtype=bool)\n",
    "    selected_features = []\n",
    "\n",
    "    while mask.sum() > 0:\n",
    "        tmp_df = tmp_df.iloc[:,mask]\n",
    "        mask = np.ones(len(tmp_df.columns), dtype=bool)\n",
    "        corr = tmp_df.corr().abs().to_numpy()\n",
    "        \n",
    "        selected_features.append(tmp_df.columns[0])\n",
    "\n",
    "        corr_with_current_feature = corr[0,:]\n",
    "        idx = np.argwhere(corr_with_current_feature > .95).flatten()\n",
    "\n",
    "        mask[idx] = False\n",
    "\n",
    "    return selected_features\n",
    "\n",
    "\n",
    "selected_features = select_features(X_df, threshold=.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_df[selected_features].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = X_df[selected_features].columns.to_list()\n",
    "for i in range(len(columns)):\n",
    "    print(f'{i+1:02d} - {columns[i]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_df[selected_features].to_csv('data/dataset/dfu_features_dataset_selected.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('DeepLearning')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "1d9b8aa8d774518be7ebcfd06a2463a8035a66798fac49b1a363f570d2d8622e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}