Feature_Selection_Hybrid_method.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayushi15092002/mental-health-workload/blob/main/Feature_Selection_Hybrid_method.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KbjRJwVEvX1c",
        "outputId": "3d58a0e7-91b6-4b55-fb6e-04cb95def03e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# Load the Drive helper and mount\n",
        "from google.colab import drive\n",
        "\n",
        "# This will prompt for authorization.\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "FgFkjgvDu5lB"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "# import seaborn as sns\n",
        "# import matplotlib.pyplot as plt\n",
        "import os\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "# plt.style.use('seaborn-colorblind')\n",
        "# %matplotlib inline\n",
        "from sklearn.feature_selection import RFE\n",
        "# from feature_selection import hybrid\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fsdUjYmRu5lF"
      },
      "source": [
        "## Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "xHNBFp0pu5lI"
      },
      "outputs": [],
      "source": [
        "data = pd.read_excel(r'/content/drive/My Drive/drdo/features/min_max_norm.xlsx')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317
        },
        "id": "Ui8HWlKxu5lJ",
        "outputId": "fbe4b603-c5e5-4c47-fe67-873a4f80a704"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Coeffiecient_of_Variation  Mean_of_Vertex_to_Vertex_Slope  \\\n",
              "0                   0.534439                        0.614380   \n",
              "1                   0.534315                        0.649929   \n",
              "2                   0.534344                        0.583582   \n",
              "3                   0.542825                        0.572106   \n",
              "4                   0.534477                        0.629300   \n",
              "\n",
              "   Variance_of_Vertex_to_Vertex_Slope  Hjorth_Activity  Hjorth_Mobility  \\\n",
              "0                            0.096161         0.020099         0.555111   \n",
              "1                            0.068544         0.025495         0.455425   \n",
              "2                            0.112827         0.027777         0.494449   \n",
              "3                            0.097661         0.033724         0.452482   \n",
              "4                            0.097666         0.029060         0.475090   \n",
              "\n",
              "   Hjorth_Complexity  Kurtosis  2nd_Difference_Mean  2nd_Difference_Max  \\\n",
              "0           0.052773  0.159801             0.248315            0.320428   \n",
              "1           0.084819  0.133192             0.202098            0.254608   \n",
              "2           0.070979  0.099703             0.281429            0.347117   \n",
              "3           0.085299  0.141227             0.260114            0.302845   \n",
              "4           0.078828  0.121195             0.253848            0.293299   \n",
              "\n",
              "   Skewness  ...  28_Wavelet_Approximate_Std_Deviation  \\\n",
              "0  0.528456  ...                              0.103340   \n",
              "1  0.543443  ...                              0.146594   \n",
              "2  0.595850  ...                              0.081346   \n",
              "3  0.450263  ...                              0.141749   \n",
              "4  0.568707  ...                              0.119258   \n",
              "\n",
              "   28_Wavelet_Approximate_Energy  28_Wavelet_Detailed_Mean  \\\n",
              "0                       0.009270                  0.384685   \n",
              "1                       0.008348                  0.571264   \n",
              "2                       0.004189                  0.484646   \n",
              "3                       0.008017                  0.776456   \n",
              "4                       0.008177                  0.550035   \n",
              "\n",
              "   28_Wavelet_Detailed_Std _Deviation  28_Wavelet_Detailed_Energy  \\\n",
              "0                            0.344930                    0.171401   \n",
              "1                            0.323252                    0.155246   \n",
              "2                            0.452760                    0.262473   \n",
              "3                            0.437634                    0.248645   \n",
              "4                            0.382984                    0.201500   \n",
              "\n",
              "   28_Wavelet_Approximate_Entropy  28_Wavelet_Detailed_Entropy  \\\n",
              "0                        0.184330                     0.011559   \n",
              "1                        0.166751                     0.010475   \n",
              "2                        0.278811                     0.005396   \n",
              "3                        0.263571                     0.010023   \n",
              "4                        0.215768                     0.010123   \n",
              "\n",
              "   Hurst_Exponent  Permutation_Entropy    y  \n",
              "0        0.723232             0.818775  1.0  \n",
              "1        0.728989             0.812935  1.0  \n",
              "2        0.751985             0.878779  1.0  \n",
              "3        0.590368             0.849525  1.0  \n",
              "4        0.612329             0.855054  1.0  \n",
              "\n",
              "[5 rows x 247 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-70c84e52-aae3-44d0-a110-f2f800323961\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Coeffiecient_of_Variation</th>\n",
              "      <th>Mean_of_Vertex_to_Vertex_Slope</th>\n",
              "      <th>Variance_of_Vertex_to_Vertex_Slope</th>\n",
              "      <th>Hjorth_Activity</th>\n",
              "      <th>Hjorth_Mobility</th>\n",
              "      <th>Hjorth_Complexity</th>\n",
              "      <th>Kurtosis</th>\n",
              "      <th>2nd_Difference_Mean</th>\n",
              "      <th>2nd_Difference_Max</th>\n",
              "      <th>Skewness</th>\n",
              "      <th>...</th>\n",
              "      <th>28_Wavelet_Approximate_Std_Deviation</th>\n",
              "      <th>28_Wavelet_Approximate_Energy</th>\n",
              "      <th>28_Wavelet_Detailed_Mean</th>\n",
              "      <th>28_Wavelet_Detailed_Std _Deviation</th>\n",
              "      <th>28_Wavelet_Detailed_Energy</th>\n",
              "      <th>28_Wavelet_Approximate_Entropy</th>\n",
              "      <th>28_Wavelet_Detailed_Entropy</th>\n",
              "      <th>Hurst_Exponent</th>\n",
              "      <th>Permutation_Entropy</th>\n",
              "      <th>y</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.534439</td>\n",
              "      <td>0.614380</td>\n",
              "      <td>0.096161</td>\n",
              "      <td>0.020099</td>\n",
              "      <td>0.555111</td>\n",
              "      <td>0.052773</td>\n",
              "      <td>0.159801</td>\n",
              "      <td>0.248315</td>\n",
              "      <td>0.320428</td>\n",
              "      <td>0.528456</td>\n",
              "      <td>...</td>\n",
              "      <td>0.103340</td>\n",
              "      <td>0.009270</td>\n",
              "      <td>0.384685</td>\n",
              "      <td>0.344930</td>\n",
              "      <td>0.171401</td>\n",
              "      <td>0.184330</td>\n",
              "      <td>0.011559</td>\n",
              "      <td>0.723232</td>\n",
              "      <td>0.818775</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.534315</td>\n",
              "      <td>0.649929</td>\n",
              "      <td>0.068544</td>\n",
              "      <td>0.025495</td>\n",
              "      <td>0.455425</td>\n",
              "      <td>0.084819</td>\n",
              "      <td>0.133192</td>\n",
              "      <td>0.202098</td>\n",
              "      <td>0.254608</td>\n",
              "      <td>0.543443</td>\n",
              "      <td>...</td>\n",
              "      <td>0.146594</td>\n",
              "      <td>0.008348</td>\n",
              "      <td>0.571264</td>\n",
              "      <td>0.323252</td>\n",
              "      <td>0.155246</td>\n",
              "      <td>0.166751</td>\n",
              "      <td>0.010475</td>\n",
              "      <td>0.728989</td>\n",
              "      <td>0.812935</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.534344</td>\n",
              "      <td>0.583582</td>\n",
              "      <td>0.112827</td>\n",
              "      <td>0.027777</td>\n",
              "      <td>0.494449</td>\n",
              "      <td>0.070979</td>\n",
              "      <td>0.099703</td>\n",
              "      <td>0.281429</td>\n",
              "      <td>0.347117</td>\n",
              "      <td>0.595850</td>\n",
              "      <td>...</td>\n",
              "      <td>0.081346</td>\n",
              "      <td>0.004189</td>\n",
              "      <td>0.484646</td>\n",
              "      <td>0.452760</td>\n",
              "      <td>0.262473</td>\n",
              "      <td>0.278811</td>\n",
              "      <td>0.005396</td>\n",
              "      <td>0.751985</td>\n",
              "      <td>0.878779</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.542825</td>\n",
              "      <td>0.572106</td>\n",
              "      <td>0.097661</td>\n",
              "      <td>0.033724</td>\n",
              "      <td>0.452482</td>\n",
              "      <td>0.085299</td>\n",
              "      <td>0.141227</td>\n",
              "      <td>0.260114</td>\n",
              "      <td>0.302845</td>\n",
              "      <td>0.450263</td>\n",
              "      <td>...</td>\n",
              "      <td>0.141749</td>\n",
              "      <td>0.008017</td>\n",
              "      <td>0.776456</td>\n",
              "      <td>0.437634</td>\n",
              "      <td>0.248645</td>\n",
              "      <td>0.263571</td>\n",
              "      <td>0.010023</td>\n",
              "      <td>0.590368</td>\n",
              "      <td>0.849525</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.534477</td>\n",
              "      <td>0.629300</td>\n",
              "      <td>0.097666</td>\n",
              "      <td>0.029060</td>\n",
              "      <td>0.475090</td>\n",
              "      <td>0.078828</td>\n",
              "      <td>0.121195</td>\n",
              "      <td>0.253848</td>\n",
              "      <td>0.293299</td>\n",
              "      <td>0.568707</td>\n",
              "      <td>...</td>\n",
              "      <td>0.119258</td>\n",
              "      <td>0.008177</td>\n",
              "      <td>0.550035</td>\n",
              "      <td>0.382984</td>\n",
              "      <td>0.201500</td>\n",
              "      <td>0.215768</td>\n",
              "      <td>0.010123</td>\n",
              "      <td>0.612329</td>\n",
              "      <td>0.855054</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 247 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-70c84e52-aae3-44d0-a110-f2f800323961')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-70c84e52-aae3-44d0-a110-f2f800323961 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-70c84e52-aae3-44d0-a110-f2f800323961');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "data.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1ck9FLhKu5lK",
        "outputId": "06cba637-4d58-4292-f512-62c1150abb60"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((9256, 246), (2314, 246))"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(data.drop(labels=['y'], axis=1), \n",
        "                                                    data.y, test_size=0.2,\n",
        "                                                    random_state=0)\n",
        "X_train.shape, X_test.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ipn7eZtCu5lL"
      },
      "source": [
        "##  Recursive Feature Elimination \n",
        "### with Random Forests Importance\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UeERZIhju5lL"
      },
      "source": [
        "### Example 1\n",
        "This method is slightly **different from the guide**, as it use a different stopping criterion: the desired number of features to select is eventually reached."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PnJkqf31u5lM",
        "outputId": "7640c749-8dec-4741-adc6-b94f28b6617c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RFE(estimator=RandomForestClassifier(n_estimators=20), n_features_to_select=10)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "#  n_features_to_select decide the stopping criterion\n",
        "# we stop till 10 features remaining\n",
        "\n",
        "sel_ = RFE(RandomForestClassifier(n_estimators=20), n_features_to_select=10)\n",
        "sel_.fit(X_train.fillna(0), y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KTs-ptbeu5lN",
        "outputId": "91f978ee-d92f-4a7b-fdd7-96dc571e4685"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Index(['Delta/Theta', '7_Wavelet_Detailed_Energy', '8_Wavelet_Detailed_Energy',\n",
            "       '10_Wavelet_Approximate_Entropy', '12_Wavelet_Detailed_Std _Deviation',\n",
            "       '20_Wavelet_Approximate_Entropy', '21_Wavelet_Approximate_Entropy',\n",
            "       '23_Wavelet_Detailed_Energy', '28_Wavelet_Approximate_Entropy',\n",
            "       'Permutation_Entropy'],\n",
            "      dtype='object')\n"
          ]
        }
      ],
      "source": [
        "selected_feat = X_train.columns[(sel_.get_support())]\n",
        "print(selected_feat)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "id": "KkEyTq0Qu5lO"
      },
      "source": [
        "### Example 2\n",
        "recursive feature elimination with RandomForest\n",
        "with the method same as the guide\n",
        "1. Rank the features according to their importance derived from a machine learning algorithm: it can be tree importance, or LASSO / Ridge,  or the linear / logistic regression coefficients.\n",
        "2. Remove one feature -the least important- and build a machine learning algorithm utilizing the remaining features.\n",
        "3. Calculate a performance metric of your choice: roc-auc, mse, rmse, accuracy.\n",
        "4. If the metric decreases by more of an arbitrarily set threshold, then that feature is important and should be kept. Otherwise, we can remove that feature.\n",
        "5. Repeat steps 2-4 until all features have been removed (and therefore evaluated) and the drop in performance assessed.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A3Qkknnru5lP",
        "outputId": "a840dc03-92dc-4234-8c9e-8ed3a0289de9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "testing feature:  Coeffiecient_of_Variation  which is feature  1  out of  246\n",
            "New Test ROC AUC=0.7368984378640789\n",
            "All features Test ROC AUC=0.7503746387547208\n",
            "Drop in ROC AUC=0.013476200890641832\n",
            "keep:  Coeffiecient_of_Variation\n",
            "\n",
            "testing feature:  Mean_of_Vertex_to_Vertex_Slope  which is feature  2  out of  246\n",
            "New Test ROC AUC=0.7498865052479988\n",
            "All features Test ROC AUC=0.7503746387547208\n",
            "Drop in ROC AUC=0.0004881335067219439\n",
            "remove:  Mean_of_Vertex_to_Vertex_Slope\n",
            "\n",
            "testing feature:  Variance_of_Vertex_to_Vertex_Slope  which is feature  3  out of  246\n",
            "New Test ROC AUC=0.7602952929735481\n",
            "All features Test ROC AUC=0.7498865052479988\n",
            "Drop in ROC AUC=-0.010408787725549318\n",
            "remove:  Variance_of_Vertex_to_Vertex_Slope\n",
            "\n",
            "testing feature:  Hjorth_Activity  which is feature  4  out of  246\n",
            "New Test ROC AUC=0.7435921568394672\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01670313613408092\n",
            "keep:  Hjorth_Activity\n",
            "\n",
            "testing feature:  Hjorth_Mobility  which is feature  5  out of  246\n",
            "New Test ROC AUC=0.7370553663621655\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.023239926611382633\n",
            "keep:  Hjorth_Mobility\n",
            "\n",
            "testing feature:  Hjorth_Complexity  which is feature  6  out of  246\n",
            "New Test ROC AUC=0.7380360150794454\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.022259277894102758\n",
            "keep:  Hjorth_Complexity\n",
            "\n",
            "testing feature:  Kurtosis  which is feature  7  out of  246\n",
            "New Test ROC AUC=0.7470246394207734\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013270653552774792\n",
            "keep:  Kurtosis\n",
            "\n",
            "testing feature:  2nd_Difference_Mean  which is feature  8  out of  246\n",
            "New Test ROC AUC=0.7502176376887795\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010077655284768672\n",
            "keep:  2nd_Difference_Mean\n",
            "\n",
            "testing feature:  2nd_Difference_Max  which is feature  9  out of  246\n",
            "New Test ROC AUC=0.7479296547260565\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012365638247491595\n",
            "keep:  2nd_Difference_Max\n",
            "\n",
            "testing feature:  Skewness  which is feature  10  out of  246\n",
            "New Test ROC AUC=0.7398308438180647\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.0204644491554834\n",
            "keep:  Skewness\n",
            "\n",
            "testing feature:  1st_Difference_Mean  which is feature  11  out of  246\n",
            "New Test ROC AUC=0.7446534464422307\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015641846531317394\n",
            "keep:  1st_Difference_Mean\n",
            "\n",
            "testing feature:  1st_Difference_Max  which is feature  12  out of  246\n",
            "New Test ROC AUC=0.7418458787175424\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01844941425600577\n",
            "keep:  1st_Difference_Max\n",
            "\n",
            "testing feature:  FFT_Delta_MaxPower  which is feature  13  out of  246\n",
            "New Test ROC AUC=0.7402719198779366\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02002337309561153\n",
            "keep:  FFT_Delta_MaxPower\n",
            "\n",
            "testing feature:  FFT_Theta_MaxPower  which is feature  14  out of  246\n",
            "New Test ROC AUC=0.7391295686579191\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02116572431562902\n",
            "keep:  FFT_Theta_MaxPower\n",
            "\n",
            "testing feature:  FFT_Alpha_MaxPower  which is feature  15  out of  246\n",
            "New Test ROC AUC=0.7484818018515372\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01181349112201091\n",
            "keep:  FFT_Alpha_MaxPower\n",
            "\n",
            "testing feature:  FFT_Beta_MaxPower  which is feature  16  out of  246\n",
            "New Test ROC AUC=0.7388954673337155\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02139982563983267\n",
            "keep:  FFT_Beta_MaxPower\n",
            "\n",
            "testing feature:  Delta/Theta  which is feature  17  out of  246\n",
            "New Test ROC AUC=0.7447499718511753\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015545321122372835\n",
            "keep:  Delta/Theta\n",
            "\n",
            "testing feature:  Delta/Alpha  which is feature  18  out of  246\n",
            "New Test ROC AUC=0.7552942768525922\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.005001016120955937\n",
            "keep:  Delta/Alpha\n",
            "\n",
            "testing feature:  Theta/Alpha  which is feature  19  out of  246\n",
            "New Test ROC AUC=0.7460664854020138\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01422880757153433\n",
            "keep:  Theta/Alpha\n",
            "\n",
            "testing feature:  (Delta+Theta)/Alpha  which is feature  20  out of  246\n",
            "New Test ROC AUC=0.7513784471394948\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008916845834053388\n",
            "keep:  (Delta+Theta)/Alpha\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Mean  which is feature  21  out of  246\n",
            "New Test ROC AUC=0.7454198431610733\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014875449812474795\n",
            "keep:  1_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Std_Deviation  which is feature  22  out of  246\n",
            "New Test ROC AUC=0.7411437705798061\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01915152239374207\n",
            "keep:  1_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Energy  which is feature  23  out of  246\n",
            "New Test ROC AUC=0.7478461747289026\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012449118244645563\n",
            "keep:  1_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Mean  which is feature  24  out of  246\n",
            "New Test ROC AUC=0.7543866599195255\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.005908633054022672\n",
            "keep:  1_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Std_Deviation  which is feature  25  out of  246\n",
            "New Test ROC AUC=0.7404675647416449\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.019827728231903263\n",
            "keep:  1_Wavelet_Detailed_Std_Deviation\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Energy  which is feature  26  out of  246\n",
            "New Test ROC AUC=0.7405310172564601\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.019764275717088053\n",
            "keep:  1_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Entropy  which is feature  27  out of  246\n",
            "New Test ROC AUC=0.7363662821080778\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02392901086547039\n",
            "keep:  1_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Entropy  which is feature  28  out of  246\n",
            "New Test ROC AUC=0.744842960582501\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015452332391047152\n",
            "keep:  1_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Mean  which is feature  29  out of  246\n",
            "New Test ROC AUC=0.7536371877415788\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.006658105231969302\n",
            "keep:  2_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Std_Deviation  which is feature  30  out of  246\n",
            "New Test ROC AUC=0.7445597050215654\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01573558795198271\n",
            "keep:  2_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Energy  which is feature  31  out of  246\n",
            "New Test ROC AUC=0.752420337464192\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.00787495550935613\n",
            "keep:  2_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Mean  which is feature  32  out of  246\n",
            "New Test ROC AUC=0.7534183300628755\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.006876962910672635\n",
            "keep:  2_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Std _Deviation  which is feature  33  out of  246\n",
            "New Test ROC AUC=0.7511718955925843\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.009123397380963794\n",
            "keep:  2_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Energy  which is feature  34  out of  246\n",
            "New Test ROC AUC=0.7511987468555067\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.009096546118041449\n",
            "keep:  2_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Entropy  which is feature  35  out of  246\n",
            "New Test ROC AUC=0.7519121132656649\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.00838317970788327\n",
            "keep:  2_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Entropy  which is feature  36  out of  246\n",
            "New Test ROC AUC=0.7559007753450601\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.004394517628488082\n",
            "keep:  2_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Mean  which is feature  37  out of  246\n",
            "New Test ROC AUC=0.7580500114903777\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.0022452814831704915\n",
            "keep:  3_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Std_Deviation  which is feature  38  out of  246\n",
            "New Test ROC AUC=0.7501647768240001\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010130516149548074\n",
            "keep:  3_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Energy  which is feature  39  out of  246\n",
            "New Test ROC AUC=0.7487493611918328\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.011545931781715368\n",
            "keep:  3_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Mean  which is feature  40  out of  246\n",
            "New Test ROC AUC=0.7531484903965916\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.007146802576956546\n",
            "keep:  3_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Std _Deviation  which is feature  41  out of  246\n",
            "New Test ROC AUC=0.7495630602918315\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010732232681716636\n",
            "keep:  3_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Energy  which is feature  42  out of  246\n",
            "New Test ROC AUC=0.7494455180287776\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010849774944770574\n",
            "keep:  3_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Entropy  which is feature  43  out of  246\n",
            "New Test ROC AUC=0.7512114985131897\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.009083794460358452\n",
            "keep:  3_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Entropy  which is feature  44  out of  246\n",
            "New Test ROC AUC=0.7428331025036542\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.017462190469893946\n",
            "keep:  3_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Mean  which is feature  45  out of  246\n",
            "New Test ROC AUC=0.7405854333802067\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.019709859593341483\n",
            "keep:  4_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Std_Deviation  which is feature  46  out of  246\n",
            "New Test ROC AUC=0.7440598900903318\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016235402883216343\n",
            "keep:  4_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Energy  which is feature  47  out of  246\n",
            "New Test ROC AUC=0.7434050710892093\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016890221884338863\n",
            "keep:  4_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Mean  which is feature  48  out of  246\n",
            "New Test ROC AUC=0.7453106855332271\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01498460744032104\n",
            "keep:  4_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Std _Deviation  which is feature  49  out of  246\n",
            "New Test ROC AUC=0.7401788839729825\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02011640900056566\n",
            "keep:  4_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Energy  which is feature  50  out of  246\n",
            "New Test ROC AUC=0.7401788839729825\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02011640900056566\n",
            "keep:  4_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Entropy  which is feature  51  out of  246\n",
            "New Test ROC AUC=0.7450220220484074\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015273270925140747\n",
            "keep:  4_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Entropy  which is feature  52  out of  246\n",
            "New Test ROC AUC=0.7438731766469736\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016422116326574532\n",
            "keep:  4_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Mean  which is feature  53  out of  246\n",
            "New Test ROC AUC=0.7396472863020294\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02064800667151878\n",
            "keep:  5_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Std_Deviation  which is feature  54  out of  246\n",
            "New Test ROC AUC=0.7520627901693049\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008232502804243258\n",
            "keep:  5_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Energy  which is feature  55  out of  246\n",
            "New Test ROC AUC=0.7522890551585414\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008006237815006756\n",
            "keep:  5_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Mean  which is feature  56  out of  246\n",
            "New Test ROC AUC=0.7550182013328256\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.005277091640722542\n",
            "keep:  5_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Std _Deviation  which is feature  57  out of  246\n",
            "New Test ROC AUC=0.7417424938577599\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01855279911578822\n",
            "keep:  5_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Energy  which is feature  58  out of  246\n",
            "New Test ROC AUC=0.7404155021173804\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.019879790856167756\n",
            "keep:  5_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Entropy  which is feature  59  out of  246\n",
            "New Test ROC AUC=0.7435518154461476\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016743477527400574\n",
            "keep:  5_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Entropy  which is feature  60  out of  246\n",
            "New Test ROC AUC=0.74827397665338\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012021316320168118\n",
            "keep:  5_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Mean  which is feature  61  out of  246\n",
            "New Test ROC AUC=0.7402601504230003\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020035142550547835\n",
            "keep:  6_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Std_Deviation  which is feature  62  out of  246\n",
            "New Test ROC AUC=0.7438422166866254\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01645307628692272\n",
            "keep:  6_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Energy  which is feature  63  out of  246\n",
            "New Test ROC AUC=0.7403202054617027\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.019975087511845402\n",
            "keep:  6_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Mean  which is feature  64  out of  246\n",
            "New Test ROC AUC=0.7355865468170282\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.024708746156519967\n",
            "keep:  6_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Std _Deviation  which is feature  65  out of  246\n",
            "New Test ROC AUC=0.7359622004390969\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02433309253445126\n",
            "keep:  6_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Energy  which is feature  66  out of  246\n",
            "New Test ROC AUC=0.7358928567742149\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.024402436199333244\n",
            "keep:  6_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Entropy  which is feature  67  out of  246\n",
            "New Test ROC AUC=0.7358040069982519\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.024491285975296262\n",
            "keep:  6_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Entropy  which is feature  68  out of  246\n",
            "New Test ROC AUC=0.7455467974526981\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014748495520850002\n",
            "keep:  6_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Mean  which is feature  69  out of  246\n",
            "New Test ROC AUC=0.7542655656307596\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.0060297273427885045\n",
            "keep:  7_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Std_Deviation  which is feature  70  out of  246\n",
            "New Test ROC AUC=0.753879836578862\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.006415456394686103\n",
            "keep:  7_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Energy  which is feature  71  out of  246\n",
            "New Test ROC AUC=0.7502924870293302\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010002805944217985\n",
            "keep:  7_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Mean  which is feature  72  out of  246\n",
            "New Test ROC AUC=0.7509407846487822\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.00935450832476592\n",
            "keep:  7_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Std _Deviation  which is feature  73  out of  246\n",
            "New Test ROC AUC=0.7422383518394543\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01805694113409384\n",
            "keep:  7_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Energy  which is feature  74  out of  246\n",
            "New Test ROC AUC=0.7422503710522672\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018044921921280932\n",
            "keep:  7_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Entropy  which is feature  75  out of  246\n",
            "New Test ROC AUC=0.7464139244993535\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013881368474194633\n",
            "keep:  7_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Entropy  which is feature  76  out of  246\n",
            "New Test ROC AUC=0.7395324047939006\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020762888179647554\n",
            "keep:  7_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Mean  which is feature  77  out of  246\n",
            "New Test ROC AUC=0.7375118268069791\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02278346616656901\n",
            "keep:  8_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Std_Deviation  which is feature  78  out of  246\n",
            "New Test ROC AUC=0.74417728273789\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016118010235658176\n",
            "keep:  8_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Energy  which is feature  79  out of  246\n",
            "New Test ROC AUC=0.7465646858030568\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013730607170491349\n",
            "keep:  8_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Mean  which is feature  80  out of  246\n",
            "New Test ROC AUC=0.7452906049591248\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01500468801442334\n",
            "keep:  8_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Std _Deviation  which is feature  81  out of  246\n",
            "New Test ROC AUC=0.7517860562524834\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008509236721064739\n",
            "keep:  8_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Energy  which is feature  82  out of  246\n",
            "New Test ROC AUC=0.7517596784753734\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008535614498174748\n",
            "keep:  8_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Entropy  which is feature  83  out of  246\n",
            "New Test ROC AUC=0.7513080538611248\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008987239112423384\n",
            "keep:  8_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Entropy  which is feature  84  out of  246\n",
            "New Test ROC AUC=0.7438915250856774\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01640376788787079\n",
            "keep:  8_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Mean  which is feature  85  out of  246\n",
            "New Test ROC AUC=0.7529838155243558\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.007311477449192361\n",
            "keep:  9_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Std_Deviation  which is feature  86  out of  246\n",
            "New Test ROC AUC=0.7435281471184575\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016767145855090648\n",
            "keep:  9_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Energy  which is feature  87  out of  246\n",
            "New Test ROC AUC=0.7453501709150742\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01494512205847398\n",
            "keep:  9_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Mean  which is feature  88  out of  246\n",
            "New Test ROC AUC=0.748954825567279\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01134046740626915\n",
            "keep:  9_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Std _Deviation  which is feature  89  out of  246\n",
            "New Test ROC AUC=0.739379803778655\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020915489194893166\n",
            "keep:  9_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Energy  which is feature  90  out of  246\n",
            "New Test ROC AUC=0.7393129546549857\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020982338318562443\n",
            "keep:  9_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Entropy  which is feature  91  out of  246\n",
            "New Test ROC AUC=0.7421967709054891\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018098522068059064\n",
            "keep:  9_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Entropy  which is feature  92  out of  246\n",
            "New Test ROC AUC=0.7495479791878936\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010747313785654522\n",
            "keep:  9_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Mean  which is feature  93  out of  246\n",
            "New Test ROC AUC=0.7478675831168016\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012427709856746572\n",
            "keep:  10_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Std_Deviation  which is feature  94  out of  246\n",
            "New Test ROC AUC=0.7422754919038903\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018019801069657837\n",
            "keep:  10_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Energy  which is feature  95  out of  246\n",
            "New Test ROC AUC=0.734241255799176\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.026054037174372158\n",
            "keep:  10_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Mean  which is feature  96  out of  246\n",
            "New Test ROC AUC=0.7447562615160668\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015539031457481367\n",
            "keep:  10_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Std _Deviation  which is feature  97  out of  246\n",
            "New Test ROC AUC=0.7404887382772735\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01980655469627468\n",
            "keep:  10_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Energy  which is feature  98  out of  246\n",
            "New Test ROC AUC=0.7404887382772735\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01980655469627468\n",
            "keep:  10_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Entropy  which is feature  99  out of  246\n",
            "New Test ROC AUC=0.7412812427076672\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01901405026588099\n",
            "keep:  10_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Entropy  which is feature  100  out of  246\n",
            "New Test ROC AUC=0.7303607167276774\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.029934576245870703\n",
            "keep:  10_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Mean  which is feature  101  out of  246\n",
            "New Test ROC AUC=0.7381552551761289\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.022140037797419243\n",
            "keep:  11_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Std_Deviation  which is feature  102  out of  246\n",
            "New Test ROC AUC=0.7497508327736329\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010544460199915284\n",
            "keep:  11_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Energy  which is feature  103  out of  246\n",
            "New Test ROC AUC=0.7491915474786515\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.011103745494896677\n",
            "keep:  11_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Mean  which is feature  104  out of  246\n",
            "New Test ROC AUC=0.7445274320015285\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015767860972019654\n",
            "keep:  11_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Std _Deviation  which is feature  105  out of  246\n",
            "New Test ROC AUC=0.7422149790837893\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018080313889758837\n",
            "keep:  11_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Energy  which is feature  106  out of  246\n",
            "New Test ROC AUC=0.7422343447587783\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01806094821476989\n",
            "keep:  11_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Entropy  which is feature  107  out of  246\n",
            "New Test ROC AUC=0.7421923684046333\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018102924568914824\n",
            "keep:  11_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Entropy  which is feature  108  out of  246\n",
            "New Test ROC AUC=0.7409615003956906\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.0193337925778575\n",
            "keep:  11_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Mean  which is feature  109  out of  246\n",
            "New Test ROC AUC=0.7450695515335982\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015225741439949925\n",
            "keep:  12_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Std_Deviation  which is feature  110  out of  246\n",
            "New Test ROC AUC=0.7457017067435545\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014593586229993627\n",
            "keep:  12_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Energy  which is feature  111  out of  246\n",
            "New Test ROC AUC=0.7407805978295097\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.019514695144038474\n",
            "keep:  12_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Mean  which is feature  112  out of  246\n",
            "New Test ROC AUC=0.7345418610231604\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.025753431950387706\n",
            "keep:  12_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Std _Deviation  which is feature  113  out of  246\n",
            "New Test ROC AUC=0.7400139109509595\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020281382022588668\n",
            "keep:  12_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Energy  which is feature  114  out of  246\n",
            "New Test ROC AUC=0.740007848202702\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020287444770846097\n",
            "keep:  12_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Entropy  which is feature  115  out of  246\n",
            "New Test ROC AUC=0.7390173958196491\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02127789715389905\n",
            "keep:  12_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Entropy  which is feature  116  out of  246\n",
            "New Test ROC AUC=0.7391643001848838\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02113099278866437\n",
            "keep:  12_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Mean  which is feature  117  out of  246\n",
            "New Test ROC AUC=0.7334127922051862\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02688250076836196\n",
            "keep:  13_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Std_Deviation  which is feature  118  out of  246\n",
            "New Test ROC AUC=0.7445362130729062\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01575907990064196\n",
            "keep:  13_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Energy  which is feature  119  out of  246\n",
            "New Test ROC AUC=0.7430447123910187\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.017250580582529484\n",
            "keep:  13_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Mean  which is feature  120  out of  246\n",
            "New Test ROC AUC=0.750848625485753\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.009446667487795102\n",
            "keep:  13_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Std _Deviation  which is feature  121  out of  246\n",
            "New Test ROC AUC=0.7418698068773165\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018425486096231625\n",
            "keep:  13_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Energy  which is feature  122  out of  246\n",
            "New Test ROC AUC=0.7418979687152187\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018397324258329406\n",
            "keep:  13_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Entropy  which is feature  123  out of  246\n",
            "New Test ROC AUC=0.7420196184506477\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018275674522900465\n",
            "keep:  13_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Entropy  which is feature  124  out of  246\n",
            "New Test ROC AUC=0.7455033886334547\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014791904340093476\n",
            "keep:  13_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Mean  which is feature  125  out of  246\n",
            "New Test ROC AUC=0.7458603777744903\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014434915199057818\n",
            "keep:  14_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Std_Deviation  which is feature  126  out of  246\n",
            "New Test ROC AUC=0.7472574521928329\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013037840780715237\n",
            "keep:  14_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Energy  which is feature  127  out of  246\n",
            "New Test ROC AUC=0.7448172747160012\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01547801825754691\n",
            "keep:  14_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Mean  which is feature  128  out of  246\n",
            "New Test ROC AUC=0.7418192806767635\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01847601229678464\n",
            "keep:  14_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Std _Deviation  which is feature  129  out of  246\n",
            "New Test ROC AUC=0.749660218016861\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01063507495668714\n",
            "keep:  14_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Energy  which is feature  130  out of  246\n",
            "New Test ROC AUC=0.7496460100692768\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010649282904271362\n",
            "keep:  14_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Entropy  which is feature  131  out of  246\n",
            "New Test ROC AUC=0.7467714391629523\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013523853810595843\n",
            "keep:  14_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Entropy  which is feature  132  out of  246\n",
            "New Test ROC AUC=0.7489801437554559\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01131514921809229\n",
            "keep:  14_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Mean  which is feature  133  out of  246\n",
            "New Test ROC AUC=0.7436265464463658\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01666874652718231\n",
            "keep:  15_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Std_Deviation  which is feature  134  out of  246\n",
            "New Test ROC AUC=0.7475804401575221\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012714852816026023\n",
            "keep:  15_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Energy  which is feature  135  out of  246\n",
            "New Test ROC AUC=0.74406694239005\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01622835058349814\n",
            "keep:  15_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Mean  which is feature  136  out of  246\n",
            "New Test ROC AUC=0.7414947579536639\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.018800535019884212\n",
            "keep:  15_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Std _Deviation  which is feature  137  out of  246\n",
            "New Test ROC AUC=0.7454835721431518\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014811720830396391\n",
            "keep:  15_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Energy  which is feature  138  out of  246\n",
            "New Test ROC AUC=0.7454851348477706\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014810158125777573\n",
            "keep:  15_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Entropy  which is feature  139  out of  246\n",
            "New Test ROC AUC=0.7458588686802833\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01443642429326486\n",
            "keep:  15_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Entropy  which is feature  140  out of  246\n",
            "New Test ROC AUC=0.7464344714448049\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013860821528743239\n",
            "keep:  15_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Mean  which is feature  141  out of  246\n",
            "New Test ROC AUC=0.7477013686164413\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012593924357106823\n",
            "keep:  16_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Std_Deviation  which is feature  142  out of  246\n",
            "New Test ROC AUC=0.7521377094509789\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008157583522569234\n",
            "keep:  16_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Energy  which is feature  143  out of  246\n",
            "New Test ROC AUC=0.7464960858023748\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013799207171173333\n",
            "keep:  16_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Mean  which is feature  144  out of  246\n",
            "New Test ROC AUC=0.7397741607048068\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020521132268741304\n",
            "keep:  16_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Std _Deviation  which is feature  145  out of  246\n",
            "New Test ROC AUC=0.7383927633627316\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.021902529610816512\n",
            "keep:  16_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Energy  which is feature  146  out of  246\n",
            "New Test ROC AUC=0.7383927633627316\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.021902529610816512\n",
            "keep:  16_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Entropy  which is feature  147  out of  246\n",
            "New Test ROC AUC=0.7428973172121627\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.017397975761385465\n",
            "keep:  16_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Entropy  which is feature  148  out of  246\n",
            "New Test ROC AUC=0.7477574210832122\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01253787189033595\n",
            "keep:  16_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Mean  which is feature  149  out of  246\n",
            "New Test ROC AUC=0.7460029231896116\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.014292369783936554\n",
            "keep:  17_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Std_Deviation  which is feature  150  out of  246\n",
            "New Test ROC AUC=0.7401815860255433\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02011370694800485\n",
            "keep:  17_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Energy  which is feature  151  out of  246\n",
            "New Test ROC AUC=0.7486592056640484\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01163608730949972\n",
            "keep:  17_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Mean  which is feature  152  out of  246\n",
            "New Test ROC AUC=0.7464706775015285\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01382461547201963\n",
            "keep:  17_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Std _Deviation  which is feature  153  out of  246\n",
            "New Test ROC AUC=0.7397098573910758\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02058543558247239\n",
            "keep:  17_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Energy  which is feature  154  out of  246\n",
            "New Test ROC AUC=0.739729084213404\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.020566208760144145\n",
            "keep:  17_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Entropy  which is feature  155  out of  246\n",
            "New Test ROC AUC=0.7369491571053275\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.023346135868220652\n",
            "keep:  17_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Entropy  which is feature  156  out of  246\n",
            "New Test ROC AUC=0.7502599550131556\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010035337960392576\n",
            "keep:  17_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Mean  which is feature  157  out of  246\n",
            "New Test ROC AUC=0.7546231652146003\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.005672127758947876\n",
            "keep:  18_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Std_Deviation  which is feature  158  out of  246\n",
            "New Test ROC AUC=0.7520304030037359\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008264889969812228\n",
            "keep:  18_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Energy  which is feature  159  out of  246\n",
            "New Test ROC AUC=0.7362177223816944\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02407757059185378\n",
            "keep:  18_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Mean  which is feature  160  out of  246\n",
            "New Test ROC AUC=0.7504027353153909\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.00989255765815722\n",
            "keep:  18_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Std _Deviation  which is feature  161  out of  246\n",
            "New Test ROC AUC=0.7425107216290372\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.017784571344510947\n",
            "keep:  18_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Energy  which is feature  162  out of  246\n",
            "New Test ROC AUC=0.7425881965298825\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.017707096443665638\n",
            "keep:  18_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Entropy  which is feature  163  out of  246\n",
            "New Test ROC AUC=0.7408988699879816\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.01939642298556654\n",
            "keep:  18_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Entropy  which is feature  164  out of  246\n",
            "New Test ROC AUC=0.7497452073106904\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.010550085662857756\n",
            "keep:  18_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Mean  which is feature  165  out of  246\n",
            "New Test ROC AUC=0.7451236375149269\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.015171655458621203\n",
            "keep:  19_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Std_Deviation  which is feature  166  out of  246\n",
            "New Test ROC AUC=0.7515073655219628\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008787927451585298\n",
            "keep:  19_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Energy  which is feature  167  out of  246\n",
            "New Test ROC AUC=0.7481298819947761\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.012165410978771996\n",
            "keep:  19_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Mean  which is feature  168  out of  246\n",
            "New Test ROC AUC=0.7468622046781062\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.013433088295441986\n",
            "keep:  19_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Std _Deviation  which is feature  169  out of  246\n",
            "New Test ROC AUC=0.7367332759136433\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.023562017059904794\n",
            "keep:  19_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Energy  which is feature  170  out of  246\n",
            "New Test ROC AUC=0.7367006520134378\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.023594640960110325\n",
            "keep:  19_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Entropy  which is feature  171  out of  246\n",
            "New Test ROC AUC=0.7383514331567249\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.02194385981682323\n",
            "keep:  19_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Entropy  which is feature  172  out of  246\n",
            "New Test ROC AUC=0.7512761329959234\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.009019159977624702\n",
            "keep:  19_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Mean  which is feature  173  out of  246\n",
            "New Test ROC AUC=0.7483105749239279\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.011984718049620202\n",
            "keep:  20_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Std_Deviation  which is feature  174  out of  246\n",
            "New Test ROC AUC=0.7440401283085366\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.016255164665011534\n",
            "keep:  20_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Energy  which is feature  175  out of  246\n",
            "New Test ROC AUC=0.75149301822726\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.008802274746288141\n",
            "keep:  20_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Mean  which is feature  176  out of  246\n",
            "New Test ROC AUC=0.754940340941043\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.005354952032505134\n",
            "keep:  20_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Std _Deviation  which is feature  177  out of  246\n",
            "New Test ROC AUC=0.7601378522447028\n",
            "All features Test ROC AUC=0.7602952929735481\n",
            "Drop in ROC AUC=0.00015744072884538163\n",
            "remove:  20_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Energy  which is feature  178  out of  246\n",
            "New Test ROC AUC=0.7417212188741832\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.018416633370519575\n",
            "keep:  20_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Entropy  which is feature  179  out of  246\n",
            "New Test ROC AUC=0.7413532803383872\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01878457190631555\n",
            "keep:  20_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Entropy  which is feature  180  out of  246\n",
            "New Test ROC AUC=0.7443094361789789\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015828416065723827\n",
            "keep:  20_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Mean  which is feature  181  out of  246\n",
            "New Test ROC AUC=0.7453702573694075\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01476759487529522\n",
            "keep:  21_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Std_Deviation  which is feature  182  out of  246\n",
            "New Test ROC AUC=0.7475103129932256\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.012627539251477193\n",
            "keep:  21_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Energy  which is feature  183  out of  246\n",
            "New Test ROC AUC=0.7434842925162366\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01665355972846616\n",
            "keep:  21_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Mean  which is feature  184  out of  246\n",
            "New Test ROC AUC=0.7344106211112397\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.02572723113346309\n",
            "keep:  21_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Std _Deviation  which is feature  185  out of  246\n",
            "New Test ROC AUC=0.7454285765080382\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014709275736664584\n",
            "keep:  21_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Energy  which is feature  186  out of  246\n",
            "New Test ROC AUC=0.7455869815989408\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014550870645761993\n",
            "keep:  21_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Entropy  which is feature  187  out of  246\n",
            "New Test ROC AUC=0.7432698477011136\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.016868004543589166\n",
            "keep:  21_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Entropy  which is feature  188  out of  246\n",
            "New Test ROC AUC=0.745385994841892\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014751857402810709\n",
            "keep:  21_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Mean  which is feature  189  out of  246\n",
            "New Test ROC AUC=0.7519090729069197\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.008228779337783099\n",
            "keep:  22_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Std_Deviation  which is feature  190  out of  246\n",
            "New Test ROC AUC=0.7490008431552548\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011137009089447925\n",
            "keep:  22_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Energy  which is feature  191  out of  246\n",
            "New Test ROC AUC=0.7481680637144035\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011969788530299286\n",
            "keep:  22_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Mean  which is feature  192  out of  246\n",
            "New Test ROC AUC=0.7480848629028253\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.012052989341877485\n",
            "keep:  22_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Std _Deviation  which is feature  193  out of  246\n",
            "New Test ROC AUC=0.7342841275492807\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.02585372469542202\n",
            "keep:  22_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Energy  which is feature  194  out of  246\n",
            "New Test ROC AUC=0.7342351003157557\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.025902751928947088\n",
            "keep:  22_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Entropy  which is feature  195  out of  246\n",
            "New Test ROC AUC=0.7359425480566013\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.024195304188101452\n",
            "keep:  22_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Entropy  which is feature  196  out of  246\n",
            "New Test ROC AUC=0.740563021445406\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.019574830799296783\n",
            "keep:  22_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Mean  which is feature  197  out of  246\n",
            "New Test ROC AUC=0.7441438453291807\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015994006915522108\n",
            "keep:  23_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Std_Deviation  which is feature  198  out of  246\n",
            "New Test ROC AUC=0.7528169734944012\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.007320878750301518\n",
            "keep:  23_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Energy  which is feature  199  out of  246\n",
            "New Test ROC AUC=0.7466790234189846\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.013458828825718139\n",
            "keep:  23_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Mean  which is feature  200  out of  246\n",
            "New Test ROC AUC=0.7473267614315809\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.012811090813121817\n",
            "keep:  23_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Std _Deviation  which is feature  201  out of  246\n",
            "New Test ROC AUC=0.7372199604599909\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.02291789178471182\n",
            "keep:  23_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Energy  which is feature  202  out of  246\n",
            "New Test ROC AUC=0.7371884412720618\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.022949410972640916\n",
            "keep:  23_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Entropy  which is feature  203  out of  246\n",
            "New Test ROC AUC=0.7414061151802297\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.018731737064473064\n",
            "keep:  23_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Entropy  which is feature  204  out of  246\n",
            "New Test ROC AUC=0.7448343231175497\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015303529127153048\n",
            "keep:  23_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Mean  which is feature  205  out of  246\n",
            "New Test ROC AUC=0.7337776747253799\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.026360177519322847\n",
            "keep:  24_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Std_Deviation  which is feature  206  out of  246\n",
            "New Test ROC AUC=0.7418778543833627\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.018259997861340027\n",
            "keep:  24_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Energy  which is feature  207  out of  246\n",
            "New Test ROC AUC=0.7374299787760037\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.02270787346869907\n",
            "keep:  24_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Mean  which is feature  208  out of  246\n",
            "New Test ROC AUC=0.7508684650928086\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.009269387151894182\n",
            "keep:  24_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Std _Deviation  which is feature  209  out of  246\n",
            "New Test ROC AUC=0.7491658401705885\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.010972012074114246\n",
            "keep:  24_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Energy  which is feature  210  out of  246\n",
            "New Test ROC AUC=0.7491658401705885\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.010972012074114246\n",
            "keep:  24_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Entropy  which is feature  211  out of  246\n",
            "New Test ROC AUC=0.7483743038578613\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01176354838684146\n",
            "keep:  24_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Entropy  which is feature  212  out of  246\n",
            "New Test ROC AUC=0.7529553883743076\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.007182463870395117\n",
            "keep:  24_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Mean  which is feature  213  out of  246\n",
            "New Test ROC AUC=0.7450565512609836\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015081300983719181\n",
            "keep:  25_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Std_Deviation  which is feature  214  out of  246\n",
            "New Test ROC AUC=0.7369591293696417\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.023178722875061064\n",
            "keep:  25_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Energy  which is feature  215  out of  246\n",
            "New Test ROC AUC=0.7519250417648183\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.008212810479884425\n",
            "keep:  25_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Mean  which is feature  216  out of  246\n",
            "New Test ROC AUC=0.7450676536320949\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015070198612607832\n",
            "keep:  25_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Std _Deviation  which is feature  217  out of  246\n",
            "New Test ROC AUC=0.7486761195748319\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011461732669870828\n",
            "keep:  25_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Energy  which is feature  218  out of  246\n",
            "New Test ROC AUC=0.7486761195748319\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011461732669870828\n",
            "keep:  25_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Entropy  which is feature  219  out of  246\n",
            "New Test ROC AUC=0.7517426926731792\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.008395159571523592\n",
            "keep:  25_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Entropy  which is feature  220  out of  246\n",
            "New Test ROC AUC=0.7434449782501877\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.016692873994515023\n",
            "keep:  25_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Mean  which is feature  221  out of  246\n",
            "New Test ROC AUC=0.7386564797461679\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.02148137249853488\n",
            "keep:  26_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Std_Deviation  which is feature  222  out of  246\n",
            "New Test ROC AUC=0.7377797673026251\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.022358084942077694\n",
            "keep:  26_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Energy  which is feature  223  out of  246\n",
            "New Test ROC AUC=0.7545100128059652\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.00562783943873757\n",
            "keep:  26_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Mean  which is feature  224  out of  246\n",
            "New Test ROC AUC=0.7459918057466821\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014146046498020648\n",
            "keep:  26_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Std _Deviation  which is feature  225  out of  246\n",
            "New Test ROC AUC=0.7482765721533209\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011861280091381876\n",
            "keep:  26_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Energy  which is feature  226  out of  246\n",
            "New Test ROC AUC=0.7472951357488887\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.012842716495814033\n",
            "keep:  26_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Entropy  which is feature  227  out of  246\n",
            "New Test ROC AUC=0.7470763019870408\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01306155025766198\n",
            "keep:  26_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Entropy  which is feature  228  out of  246\n",
            "New Test ROC AUC=0.7439349694193805\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01620288282532223\n",
            "keep:  26_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Mean  which is feature  229  out of  246\n",
            "New Test ROC AUC=0.7443284229262389\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015809429318463852\n",
            "keep:  27_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Std_Deviation  which is feature  230  out of  246\n",
            "New Test ROC AUC=0.7461158198957496\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014022032348953162\n",
            "keep:  27_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Energy  which is feature  231  out of  246\n",
            "New Test ROC AUC=0.7371232060189135\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.023014646225789304\n",
            "keep:  27_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Mean  which is feature  232  out of  246\n",
            "New Test ROC AUC=0.7488909943271441\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01124685791755864\n",
            "keep:  27_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Std _Deviation  which is feature  233  out of  246\n",
            "New Test ROC AUC=0.7459128203837919\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014225031860910842\n",
            "keep:  27_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Energy  which is feature  234  out of  246\n",
            "New Test ROC AUC=0.7452649857657697\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.014872866478933044\n",
            "keep:  27_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Entropy  which is feature  235  out of  246\n",
            "New Test ROC AUC=0.7490749648866307\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011062887358072038\n",
            "keep:  27_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Entropy  which is feature  236  out of  246\n",
            "New Test ROC AUC=0.7359597971934626\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.024178055051240155\n",
            "keep:  27_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Mean  which is feature  237  out of  246\n",
            "New Test ROC AUC=0.7405473200491128\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.019590532195589927\n",
            "keep:  28_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Std_Deviation  which is feature  238  out of  246\n",
            "New Test ROC AUC=0.75071184252441\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.009426009720292727\n",
            "keep:  28_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Energy  which is feature  239  out of  246\n",
            "New Test ROC AUC=0.7483976045838603\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.011740247660842473\n",
            "keep:  28_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Mean  which is feature  240  out of  246\n",
            "New Test ROC AUC=0.7503551896900675\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.009782662554635224\n",
            "keep:  28_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Std _Deviation  which is feature  241  out of  246\n",
            "New Test ROC AUC=0.7434464318077684\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01669142043693439\n",
            "keep:  28_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Energy  which is feature  242  out of  246\n",
            "New Test ROC AUC=0.7434061243382969\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01673172790640587\n",
            "keep:  28_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Entropy  which is feature  243  out of  246\n",
            "New Test ROC AUC=0.7468388767076509\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.013298975537051838\n",
            "keep:  28_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Entropy  which is feature  244  out of  246\n",
            "New Test ROC AUC=0.7431269604478553\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01701089179684745\n",
            "keep:  28_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  Hurst_Exponent  which is feature  245  out of  246\n",
            "New Test ROC AUC=0.7440563376707482\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.01608151457395457\n",
            "keep:  Hurst_Exponent\n",
            "\n",
            "testing feature:  Permutation_Entropy  which is feature  246  out of  246\n",
            "New Test ROC AUC=0.744633137613085\n",
            "All features Test ROC AUC=0.7601378522447028\n",
            "Drop in ROC AUC=0.015504714631617733\n",
            "keep:  Permutation_Entropy\n",
            "DONE!!\n",
            "total features to remove:  3\n",
            "total features to keep:  243\n"
          ]
        }
      ],
      "source": [
        "# tol decide whether we should drop or keep the feature in current round\n",
        "features_to_keep = recursive_feature_elimination_rf(X_train=X_train,\n",
        "                                                           y_train=y_train,\n",
        "                                                           X_test=X_test,\n",
        "                                                           y_test=y_test,\n",
        "                                                           tol=0.001)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IuQU5HVhu5lQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aafc5556-4279-4abc-b1be-983e4efd165c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Coeffiecient_of_Variation',\n",
              " 'Hjorth_Activity',\n",
              " 'Hjorth_Mobility',\n",
              " 'Hjorth_Complexity',\n",
              " 'Kurtosis',\n",
              " '2nd_Difference_Mean',\n",
              " '2nd_Difference_Max',\n",
              " 'Skewness',\n",
              " '1st_Difference_Mean',\n",
              " '1st_Difference_Max',\n",
              " 'FFT_Delta_MaxPower',\n",
              " 'FFT_Theta_MaxPower',\n",
              " 'FFT_Alpha_MaxPower',\n",
              " 'FFT_Beta_MaxPower',\n",
              " 'Delta/Theta',\n",
              " 'Delta/Alpha',\n",
              " 'Theta/Alpha',\n",
              " '(Delta+Theta)/Alpha',\n",
              " '1_Wavelet_Approximate_Mean',\n",
              " '1_Wavelet_Approximate_Std_Deviation',\n",
              " '1_Wavelet_Approximate_Energy',\n",
              " '1_Wavelet_Detailed_Mean',\n",
              " '1_Wavelet_Detailed_Std_Deviation',\n",
              " '1_Wavelet_Detailed_Energy',\n",
              " '1_Wavelet_Approximate_Entropy',\n",
              " '1_Wavelet_Detailed_Entropy',\n",
              " '2_Wavelet_Approximate_Mean',\n",
              " '2_Wavelet_Approximate_Std_Deviation',\n",
              " '2_Wavelet_Approximate_Energy',\n",
              " '2_Wavelet_Detailed_Mean',\n",
              " '2_Wavelet_Detailed_Std _Deviation',\n",
              " '2_Wavelet_Detailed_Energy',\n",
              " '2_Wavelet_Approximate_Entropy',\n",
              " '2_Wavelet_Detailed_Entropy',\n",
              " '3_Wavelet_Approximate_Mean',\n",
              " '3_Wavelet_Approximate_Std_Deviation',\n",
              " '3_Wavelet_Approximate_Energy',\n",
              " '3_Wavelet_Detailed_Mean',\n",
              " '3_Wavelet_Detailed_Std _Deviation',\n",
              " '3_Wavelet_Detailed_Energy',\n",
              " '3_Wavelet_Approximate_Entropy',\n",
              " '3_Wavelet_Detailed_Entropy',\n",
              " '4_Wavelet_Approximate_Mean',\n",
              " '4_Wavelet_Approximate_Std_Deviation',\n",
              " '4_Wavelet_Approximate_Energy',\n",
              " '4_Wavelet_Detailed_Mean',\n",
              " '4_Wavelet_Detailed_Std _Deviation',\n",
              " '4_Wavelet_Detailed_Energy',\n",
              " '4_Wavelet_Approximate_Entropy',\n",
              " '4_Wavelet_Detailed_Entropy',\n",
              " '5_Wavelet_Approximate_Mean',\n",
              " '5_Wavelet_Approximate_Std_Deviation',\n",
              " '5_Wavelet_Approximate_Energy',\n",
              " '5_Wavelet_Detailed_Mean',\n",
              " '5_Wavelet_Detailed_Std _Deviation',\n",
              " '5_Wavelet_Detailed_Energy',\n",
              " '5_Wavelet_Approximate_Entropy',\n",
              " '5_Wavelet_Detailed_Entropy',\n",
              " '6_Wavelet_Approximate_Mean',\n",
              " '6_Wavelet_Approximate_Std_Deviation',\n",
              " '6_Wavelet_Approximate_Energy',\n",
              " '6_Wavelet_Detailed_Mean',\n",
              " '6_Wavelet_Detailed_Std _Deviation',\n",
              " '6_Wavelet_Detailed_Energy',\n",
              " '6_Wavelet_Approximate_Entropy',\n",
              " '6_Wavelet_Detailed_Entropy',\n",
              " '7_Wavelet_Approximate_Mean',\n",
              " '7_Wavelet_Approximate_Std_Deviation',\n",
              " '7_Wavelet_Approximate_Energy',\n",
              " '7_Wavelet_Detailed_Mean',\n",
              " '7_Wavelet_Detailed_Std _Deviation',\n",
              " '7_Wavelet_Detailed_Energy',\n",
              " '7_Wavelet_Approximate_Entropy',\n",
              " '7_Wavelet_Detailed_Entropy',\n",
              " '8_Wavelet_Approximate_Mean',\n",
              " '8_Wavelet_Approximate_Std_Deviation',\n",
              " '8_Wavelet_Approximate_Energy',\n",
              " '8_Wavelet_Detailed_Mean',\n",
              " '8_Wavelet_Detailed_Std _Deviation',\n",
              " '8_Wavelet_Detailed_Energy',\n",
              " '8_Wavelet_Approximate_Entropy',\n",
              " '8_Wavelet_Detailed_Entropy',\n",
              " '9_Wavelet_Approximate_Mean',\n",
              " '9_Wavelet_Approximate_Std_Deviation',\n",
              " '9_Wavelet_Approximate_Energy',\n",
              " '9_Wavelet_Detailed_Mean',\n",
              " '9_Wavelet_Detailed_Std _Deviation',\n",
              " '9_Wavelet_Detailed_Energy',\n",
              " '9_Wavelet_Approximate_Entropy',\n",
              " '9_Wavelet_Detailed_Entropy',\n",
              " '10_Wavelet_Approximate_Mean',\n",
              " '10_Wavelet_Approximate_Std_Deviation',\n",
              " '10_Wavelet_Approximate_Energy',\n",
              " '10_Wavelet_Detailed_Mean',\n",
              " '10_Wavelet_Detailed_Std _Deviation',\n",
              " '10_Wavelet_Detailed_Energy',\n",
              " '10_Wavelet_Approximate_Entropy',\n",
              " '10_Wavelet_Detailed_Entropy',\n",
              " '11_Wavelet_Approximate_Mean',\n",
              " '11_Wavelet_Approximate_Std_Deviation',\n",
              " '11_Wavelet_Approximate_Energy',\n",
              " '11_Wavelet_Detailed_Mean',\n",
              " '11_Wavelet_Detailed_Std _Deviation',\n",
              " '11_Wavelet_Detailed_Energy',\n",
              " '11_Wavelet_Approximate_Entropy',\n",
              " '11_Wavelet_Detailed_Entropy',\n",
              " '12_Wavelet_Approximate_Mean',\n",
              " '12_Wavelet_Approximate_Std_Deviation',\n",
              " '12_Wavelet_Approximate_Energy',\n",
              " '12_Wavelet_Detailed_Mean',\n",
              " '12_Wavelet_Detailed_Std _Deviation',\n",
              " '12_Wavelet_Detailed_Energy',\n",
              " '12_Wavelet_Approximate_Entropy',\n",
              " '12_Wavelet_Detailed_Entropy',\n",
              " '13_Wavelet_Approximate_Mean',\n",
              " '13_Wavelet_Approximate_Std_Deviation',\n",
              " '13_Wavelet_Approximate_Energy',\n",
              " '13_Wavelet_Detailed_Mean',\n",
              " '13_Wavelet_Detailed_Std _Deviation',\n",
              " '13_Wavelet_Detailed_Energy',\n",
              " '13_Wavelet_Approximate_Entropy',\n",
              " '13_Wavelet_Detailed_Entropy',\n",
              " '14_Wavelet_Approximate_Mean',\n",
              " '14_Wavelet_Approximate_Std_Deviation',\n",
              " '14_Wavelet_Approximate_Energy',\n",
              " '14_Wavelet_Detailed_Mean',\n",
              " '14_Wavelet_Detailed_Std _Deviation',\n",
              " '14_Wavelet_Detailed_Energy',\n",
              " '14_Wavelet_Approximate_Entropy',\n",
              " '14_Wavelet_Detailed_Entropy',\n",
              " '15_Wavelet_Approximate_Mean',\n",
              " '15_Wavelet_Approximate_Std_Deviation',\n",
              " '15_Wavelet_Approximate_Energy',\n",
              " '15_Wavelet_Detailed_Mean',\n",
              " '15_Wavelet_Detailed_Std _Deviation',\n",
              " '15_Wavelet_Detailed_Energy',\n",
              " '15_Wavelet_Approximate_Entropy',\n",
              " '15_Wavelet_Detailed_Entropy',\n",
              " '16_Wavelet_Approximate_Mean',\n",
              " '16_Wavelet_Approximate_Std_Deviation',\n",
              " '16_Wavelet_Approximate_Energy',\n",
              " '16_Wavelet_Detailed_Mean',\n",
              " '16_Wavelet_Detailed_Std _Deviation',\n",
              " '16_Wavelet_Detailed_Energy',\n",
              " '16_Wavelet_Approximate_Entropy',\n",
              " '16_Wavelet_Detailed_Entropy',\n",
              " '17_Wavelet_Approximate_Mean',\n",
              " '17_Wavelet_Approximate_Std_Deviation',\n",
              " '17_Wavelet_Approximate_Energy',\n",
              " '17_Wavelet_Detailed_Mean',\n",
              " '17_Wavelet_Detailed_Std _Deviation',\n",
              " '17_Wavelet_Detailed_Energy',\n",
              " '17_Wavelet_Approximate_Entropy',\n",
              " '17_Wavelet_Detailed_Entropy',\n",
              " '18_Wavelet_Approximate_Mean',\n",
              " '18_Wavelet_Approximate_Std_Deviation',\n",
              " '18_Wavelet_Approximate_Energy',\n",
              " '18_Wavelet_Detailed_Mean',\n",
              " '18_Wavelet_Detailed_Std _Deviation',\n",
              " '18_Wavelet_Detailed_Energy',\n",
              " '18_Wavelet_Approximate_Entropy',\n",
              " '18_Wavelet_Detailed_Entropy',\n",
              " '19_Wavelet_Approximate_Mean',\n",
              " '19_Wavelet_Approximate_Std_Deviation',\n",
              " '19_Wavelet_Approximate_Energy',\n",
              " '19_Wavelet_Detailed_Mean',\n",
              " '19_Wavelet_Detailed_Std _Deviation',\n",
              " '19_Wavelet_Detailed_Energy',\n",
              " '19_Wavelet_Approximate_Entropy',\n",
              " '19_Wavelet_Detailed_Entropy',\n",
              " '20_Wavelet_Approximate_Mean',\n",
              " '20_Wavelet_Approximate_Std_Deviation',\n",
              " '20_Wavelet_Approximate_Energy',\n",
              " '20_Wavelet_Detailed_Mean',\n",
              " '20_Wavelet_Detailed_Energy',\n",
              " '20_Wavelet_Approximate_Entropy',\n",
              " '20_Wavelet_Detailed_Entropy',\n",
              " '21_Wavelet_Approximate_Mean',\n",
              " '21_Wavelet_Approximate_Std_Deviation',\n",
              " '21_Wavelet_Approximate_Energy',\n",
              " '21_Wavelet_Detailed_Mean',\n",
              " '21_Wavelet_Detailed_Std _Deviation',\n",
              " '21_Wavelet_Detailed_Energy',\n",
              " '21_Wavelet_Approximate_Entropy',\n",
              " '21_Wavelet_Detailed_Entropy',\n",
              " '22_Wavelet_Approximate_Mean',\n",
              " '22_Wavelet_Approximate_Std_Deviation',\n",
              " '22_Wavelet_Approximate_Energy',\n",
              " '22_Wavelet_Detailed_Mean',\n",
              " '22_Wavelet_Detailed_Std _Deviation',\n",
              " '22_Wavelet_Detailed_Energy',\n",
              " '22_Wavelet_Approximate_Entropy',\n",
              " '22_Wavelet_Detailed_Entropy',\n",
              " '23_Wavelet_Approximate_Mean',\n",
              " '23_Wavelet_Approximate_Std_Deviation',\n",
              " '23_Wavelet_Approximate_Energy',\n",
              " '23_Wavelet_Detailed_Mean',\n",
              " '23_Wavelet_Detailed_Std _Deviation',\n",
              " '23_Wavelet_Detailed_Energy',\n",
              " '23_Wavelet_Approximate_Entropy',\n",
              " '23_Wavelet_Detailed_Entropy',\n",
              " '24_Wavelet_Approximate_Mean',\n",
              " '24_Wavelet_Approximate_Std_Deviation',\n",
              " '24_Wavelet_Approximate_Energy',\n",
              " '24_Wavelet_Detailed_Mean',\n",
              " '24_Wavelet_Detailed_Std _Deviation',\n",
              " '24_Wavelet_Detailed_Energy',\n",
              " '24_Wavelet_Approximate_Entropy',\n",
              " '24_Wavelet_Detailed_Entropy',\n",
              " '25_Wavelet_Approximate_Mean',\n",
              " '25_Wavelet_Approximate_Std_Deviation',\n",
              " '25_Wavelet_Approximate_Energy',\n",
              " '25_Wavelet_Detailed_Mean',\n",
              " '25_Wavelet_Detailed_Std _Deviation',\n",
              " '25_Wavelet_Detailed_Energy',\n",
              " '25_Wavelet_Approximate_Entropy',\n",
              " '25_Wavelet_Detailed_Entropy',\n",
              " '26_Wavelet_Approximate_Mean',\n",
              " '26_Wavelet_Approximate_Std_Deviation',\n",
              " '26_Wavelet_Approximate_Energy',\n",
              " '26_Wavelet_Detailed_Mean',\n",
              " '26_Wavelet_Detailed_Std _Deviation',\n",
              " '26_Wavelet_Detailed_Energy',\n",
              " '26_Wavelet_Approximate_Entropy',\n",
              " '26_Wavelet_Detailed_Entropy',\n",
              " '27_Wavelet_Approximate_Mean',\n",
              " '27_Wavelet_Approximate_Std_Deviation',\n",
              " '27_Wavelet_Approximate_Energy',\n",
              " '27_Wavelet_Detailed_Mean',\n",
              " '27_Wavelet_Detailed_Std _Deviation',\n",
              " '27_Wavelet_Detailed_Energy',\n",
              " '27_Wavelet_Approximate_Entropy',\n",
              " '27_Wavelet_Detailed_Entropy',\n",
              " '28_Wavelet_Approximate_Mean',\n",
              " '28_Wavelet_Approximate_Std_Deviation',\n",
              " '28_Wavelet_Approximate_Energy',\n",
              " '28_Wavelet_Detailed_Mean',\n",
              " '28_Wavelet_Detailed_Std _Deviation',\n",
              " '28_Wavelet_Detailed_Energy',\n",
              " '28_Wavelet_Approximate_Entropy',\n",
              " '28_Wavelet_Detailed_Entropy',\n",
              " 'Hurst_Exponent',\n",
              " 'Permutation_Entropy']"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "features_to_keep"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EBWc4Y8ku5lQ"
      },
      "source": [
        "## Recursive Feature Addition\n",
        "### with Random Forests Importance"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8tmx6xHLu5lR"
      },
      "source": [
        "### Example 1\n",
        "recursive feature addition with RandomForest\n",
        "with the method same as the guide\n",
        "1. Rank the features according to their importance derived from a  machine learning algorithm: it can be tree importance, or LASSO / Ridge,  or the linear / logistic regression coefficients.\n",
        "2. Build a machine learning model with only 1 feature, the most important one, and calculate the model metric for performance.\n",
        "3. Add one feature -the most important- and build a machine learning  algorithm utilizing the added and any feature from previous rounds.\n",
        "4. Calculate a performance metric of your choice: roc-auc, mse, rmse, accuracy.\n",
        "5. If the metric increases by more than an arbitrarily set threshold,  then that feature is important and should be kept. Otherwise, we can  remove that feature.\n",
        "6. Repeat steps 2-5 until all features have been removed (and therefore evaluated) and the drop in performance assessed.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "58LxAbN8u5lS",
        "outputId": "6405db65-0c6b-438f-809d-9fff5e37659c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "testing feature:  Mean_of_Vertex_to_Vertex_Slope  which is feature  1  out of  246\n",
            "New Test ROC AUC=0.5035143245808468\n",
            "All features Test ROC AUC=0.49189948724650573\n",
            "Increase in ROC AUC=0.011614837334341044\n",
            "keep:  Mean_of_Vertex_to_Vertex_Slope\n",
            "\n",
            "testing feature:  Variance_of_Vertex_to_Vertex_Slope  which is feature  2  out of  246\n",
            "New Test ROC AUC=0.5111411310726159\n",
            "All features Test ROC AUC=0.5035143245808468\n",
            "Increase in ROC AUC=0.007626806491769167\n",
            "keep:  Variance_of_Vertex_to_Vertex_Slope\n",
            "\n",
            "testing feature:  Hjorth_Activity  which is feature  3  out of  246\n",
            "New Test ROC AUC=0.5334201088688635\n",
            "All features Test ROC AUC=0.5111411310726159\n",
            "Increase in ROC AUC=0.022278977796247545\n",
            "keep:  Hjorth_Activity\n",
            "\n",
            "testing feature:  Hjorth_Mobility  which is feature  4  out of  246\n",
            "New Test ROC AUC=0.5549981535512211\n",
            "All features Test ROC AUC=0.5334201088688635\n",
            "Increase in ROC AUC=0.021578044682357578\n",
            "keep:  Hjorth_Mobility\n",
            "\n",
            "testing feature:  Hjorth_Complexity  which is feature  5  out of  246\n",
            "New Test ROC AUC=0.5927189136920541\n",
            "All features Test ROC AUC=0.5549981535512211\n",
            "Increase in ROC AUC=0.03772076014083303\n",
            "keep:  Hjorth_Complexity\n",
            "\n",
            "testing feature:  Kurtosis  which is feature  6  out of  246\n",
            "New Test ROC AUC=0.5961384385980627\n",
            "All features Test ROC AUC=0.5927189136920541\n",
            "Increase in ROC AUC=0.0034195249060086086\n",
            "keep:  Kurtosis\n",
            "\n",
            "testing feature:  2nd_Difference_Mean  which is feature  7  out of  246\n",
            "New Test ROC AUC=0.6048347339771926\n",
            "All features Test ROC AUC=0.5961384385980627\n",
            "Increase in ROC AUC=0.00869629537912986\n",
            "keep:  2nd_Difference_Mean\n",
            "\n",
            "testing feature:  2nd_Difference_Max  which is feature  8  out of  246\n",
            "New Test ROC AUC=0.6104968494715178\n",
            "All features Test ROC AUC=0.6048347339771926\n",
            "Increase in ROC AUC=0.0056621154943252705\n",
            "keep:  2nd_Difference_Max\n",
            "\n",
            "testing feature:  Skewness  which is feature  9  out of  246\n",
            "New Test ROC AUC=0.6102398773256028\n",
            "All features Test ROC AUC=0.6104968494715178\n",
            "Increase in ROC AUC=-0.00025697214591502426\n",
            "remove:  Skewness\n",
            "\n",
            "testing feature:  1st_Difference_Mean  which is feature  10  out of  246\n",
            "New Test ROC AUC=0.6122683728095345\n",
            "All features Test ROC AUC=0.6104968494715178\n",
            "Increase in ROC AUC=0.0017715233380166673\n",
            "keep:  1st_Difference_Mean\n",
            "\n",
            "testing feature:  1st_Difference_Max  which is feature  11  out of  246\n",
            "New Test ROC AUC=0.6120450549341784\n",
            "All features Test ROC AUC=0.6122683728095345\n",
            "Increase in ROC AUC=-0.00022331787535612158\n",
            "remove:  1st_Difference_Max\n",
            "\n",
            "testing feature:  FFT_Delta_MaxPower  which is feature  12  out of  246\n",
            "New Test ROC AUC=0.6159212635539099\n",
            "All features Test ROC AUC=0.6122683728095345\n",
            "Increase in ROC AUC=0.003652890744375359\n",
            "keep:  FFT_Delta_MaxPower\n",
            "\n",
            "testing feature:  FFT_Theta_MaxPower  which is feature  13  out of  246\n",
            "New Test ROC AUC=0.633799347941738\n",
            "All features Test ROC AUC=0.6159212635539099\n",
            "Increase in ROC AUC=0.01787808438782812\n",
            "keep:  FFT_Theta_MaxPower\n",
            "\n",
            "testing feature:  FFT_Alpha_MaxPower  which is feature  14  out of  246\n",
            "New Test ROC AUC=0.6496134025033732\n",
            "All features Test ROC AUC=0.633799347941738\n",
            "Increase in ROC AUC=0.015814054561635205\n",
            "keep:  FFT_Alpha_MaxPower\n",
            "\n",
            "testing feature:  FFT_Beta_MaxPower  which is feature  15  out of  246\n",
            "New Test ROC AUC=0.6623260573520761\n",
            "All features Test ROC AUC=0.6496134025033732\n",
            "Increase in ROC AUC=0.012712654848702876\n",
            "keep:  FFT_Beta_MaxPower\n",
            "\n",
            "testing feature:  Delta/Theta  which is feature  16  out of  246\n",
            "New Test ROC AUC=0.6571978657964431\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.005128191555632933\n",
            "remove:  Delta/Theta\n",
            "\n",
            "testing feature:  Delta/Alpha  which is feature  17  out of  246\n",
            "New Test ROC AUC=0.6632717364987151\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=0.0009456791466390113\n",
            "remove:  Delta/Alpha\n",
            "\n",
            "testing feature:  Theta/Alpha  which is feature  18  out of  246\n",
            "New Test ROC AUC=0.6621045377872737\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.00022151956480231938\n",
            "remove:  Theta/Alpha\n",
            "\n",
            "testing feature:  (Delta+Theta)/Alpha  which is feature  19  out of  246\n",
            "New Test ROC AUC=0.656549958715803\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.0057760986362730415\n",
            "remove:  (Delta+Theta)/Alpha\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Mean  which is feature  20  out of  246\n",
            "New Test ROC AUC=0.6482743338981141\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.014051723453961995\n",
            "remove:  1_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Std_Deviation  which is feature  21  out of  246\n",
            "New Test ROC AUC=0.6547523937232053\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.007573663628870797\n",
            "remove:  1_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Energy  which is feature  22  out of  246\n",
            "New Test ROC AUC=0.6591206443973405\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.0032054129547355315\n",
            "remove:  1_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Mean  which is feature  23  out of  246\n",
            "New Test ROC AUC=0.6417022870758622\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=-0.02062377027621387\n",
            "remove:  1_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Std_Deviation  which is feature  24  out of  246\n",
            "New Test ROC AUC=0.6810652836115952\n",
            "All features Test ROC AUC=0.6623260573520761\n",
            "Increase in ROC AUC=0.01873922625951918\n",
            "keep:  1_Wavelet_Detailed_Std_Deviation\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Energy  which is feature  25  out of  246\n",
            "New Test ROC AUC=0.693209640330545\n",
            "All features Test ROC AUC=0.6810652836115952\n",
            "Increase in ROC AUC=0.012144356718949756\n",
            "keep:  1_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  1_Wavelet_Approximate_Entropy  which is feature  26  out of  246\n",
            "New Test ROC AUC=0.6828501664781651\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=-0.010359473852379897\n",
            "remove:  1_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  1_Wavelet_Detailed_Entropy  which is feature  27  out of  246\n",
            "New Test ROC AUC=0.6803737485724479\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=-0.012835891758097073\n",
            "remove:  1_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Mean  which is feature  28  out of  246\n",
            "New Test ROC AUC=0.6803592104318876\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=-0.012850429898657367\n",
            "remove:  2_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Std_Deviation  which is feature  29  out of  246\n",
            "New Test ROC AUC=0.6822959258785142\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=-0.01091371445203082\n",
            "remove:  2_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Energy  which is feature  30  out of  246\n",
            "New Test ROC AUC=0.6883267549438181\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=-0.004882885386726876\n",
            "remove:  2_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Mean  which is feature  31  out of  246\n",
            "New Test ROC AUC=0.6744442309282834\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=-0.018765409402261568\n",
            "remove:  2_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Std _Deviation  which is feature  32  out of  246\n",
            "New Test ROC AUC=0.6972910299634741\n",
            "All features Test ROC AUC=0.693209640330545\n",
            "Increase in ROC AUC=0.004081389632929144\n",
            "keep:  2_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Energy  which is feature  33  out of  246\n",
            "New Test ROC AUC=0.6992600821036332\n",
            "All features Test ROC AUC=0.6972910299634741\n",
            "Increase in ROC AUC=0.0019690521401590777\n",
            "keep:  2_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  2_Wavelet_Approximate_Entropy  which is feature  34  out of  246\n",
            "New Test ROC AUC=0.6972686570151693\n",
            "All features Test ROC AUC=0.6992600821036332\n",
            "Increase in ROC AUC=-0.0019914250884639495\n",
            "remove:  2_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  2_Wavelet_Detailed_Entropy  which is feature  35  out of  246\n",
            "New Test ROC AUC=0.6920072390184064\n",
            "All features Test ROC AUC=0.6992600821036332\n",
            "Increase in ROC AUC=-0.007252843085226779\n",
            "remove:  2_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Mean  which is feature  36  out of  246\n",
            "New Test ROC AUC=0.7013808880974287\n",
            "All features Test ROC AUC=0.6992600821036332\n",
            "Increase in ROC AUC=0.0021208059937954937\n",
            "keep:  3_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Std_Deviation  which is feature  37  out of  246\n",
            "New Test ROC AUC=0.6867139155297262\n",
            "All features Test ROC AUC=0.7013808880974287\n",
            "Increase in ROC AUC=-0.014666972567702552\n",
            "remove:  3_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Energy  which is feature  38  out of  246\n",
            "New Test ROC AUC=0.6989496423208714\n",
            "All features Test ROC AUC=0.7013808880974287\n",
            "Increase in ROC AUC=-0.0024312457765572892\n",
            "remove:  3_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Mean  which is feature  39  out of  246\n",
            "New Test ROC AUC=0.6906921781215751\n",
            "All features Test ROC AUC=0.7013808880974287\n",
            "Increase in ROC AUC=-0.010688709975853583\n",
            "remove:  3_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Std _Deviation  which is feature  40  out of  246\n",
            "New Test ROC AUC=0.7052730336950808\n",
            "All features Test ROC AUC=0.7013808880974287\n",
            "Increase in ROC AUC=0.0038921455976520525\n",
            "keep:  3_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Energy  which is feature  41  out of  246\n",
            "New Test ROC AUC=0.7114157760882244\n",
            "All features Test ROC AUC=0.7052730336950808\n",
            "Increase in ROC AUC=0.006142742393143608\n",
            "keep:  3_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  3_Wavelet_Approximate_Entropy  which is feature  42  out of  246\n",
            "New Test ROC AUC=0.7103448098552659\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=-0.0010709662329584368\n",
            "remove:  3_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  3_Wavelet_Detailed_Entropy  which is feature  43  out of  246\n",
            "New Test ROC AUC=0.7107128825036703\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=-0.0007028935845541096\n",
            "remove:  3_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Mean  which is feature  44  out of  246\n",
            "New Test ROC AUC=0.6948391007327863\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=-0.016576675355438053\n",
            "remove:  4_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Std_Deviation  which is feature  45  out of  246\n",
            "New Test ROC AUC=0.7112196677300997\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=-0.0001961083581246692\n",
            "remove:  4_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Energy  which is feature  46  out of  246\n",
            "New Test ROC AUC=0.7088833730911045\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=-0.002532402997119876\n",
            "remove:  4_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Mean  which is feature  47  out of  246\n",
            "New Test ROC AUC=0.7075274033249108\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=-0.0038883727633135523\n",
            "remove:  4_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Std _Deviation  which is feature  48  out of  246\n",
            "New Test ROC AUC=0.7179669059923953\n",
            "All features Test ROC AUC=0.7114157760882244\n",
            "Increase in ROC AUC=0.006551129904170949\n",
            "keep:  4_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Energy  which is feature  49  out of  246\n",
            "New Test ROC AUC=0.7071505293843782\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=-0.010816376608017153\n",
            "remove:  4_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  4_Wavelet_Approximate_Entropy  which is feature  50  out of  246\n",
            "New Test ROC AUC=0.7093434374036822\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=-0.008623468588713101\n",
            "remove:  4_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  4_Wavelet_Detailed_Entropy  which is feature  51  out of  246\n",
            "New Test ROC AUC=0.7100607323674443\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=-0.007906173624951052\n",
            "remove:  4_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Mean  which is feature  52  out of  246\n",
            "New Test ROC AUC=0.7084546831482804\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=-0.009512222844114904\n",
            "remove:  5_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Std_Deviation  which is feature  53  out of  246\n",
            "New Test ROC AUC=0.7181608782425418\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=0.00019397225014650132\n",
            "remove:  5_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Energy  which is feature  54  out of  246\n",
            "New Test ROC AUC=0.7058379108725289\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=-0.012128995119866426\n",
            "remove:  5_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Mean  which is feature  55  out of  246\n",
            "New Test ROC AUC=0.6990806098272534\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=-0.018886296165141925\n",
            "remove:  5_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Std _Deviation  which is feature  56  out of  246\n",
            "New Test ROC AUC=0.7193067276333088\n",
            "All features Test ROC AUC=0.7179669059923953\n",
            "Increase in ROC AUC=0.0013398216409135077\n",
            "keep:  5_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Energy  which is feature  57  out of  246\n",
            "New Test ROC AUC=0.7286292158455903\n",
            "All features Test ROC AUC=0.7193067276333088\n",
            "Increase in ROC AUC=0.009322488212281432\n",
            "keep:  5_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  5_Wavelet_Approximate_Entropy  which is feature  58  out of  246\n",
            "New Test ROC AUC=0.7390559275125168\n",
            "All features Test ROC AUC=0.7286292158455903\n",
            "Increase in ROC AUC=0.010426711666926547\n",
            "keep:  5_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  5_Wavelet_Detailed_Entropy  which is feature  59  out of  246\n",
            "New Test ROC AUC=0.73028960063451\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.008766326878006825\n",
            "remove:  5_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Mean  which is feature  60  out of  246\n",
            "New Test ROC AUC=0.7296313406294278\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.009424586883089003\n",
            "remove:  6_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Std_Deviation  which is feature  61  out of  246\n",
            "New Test ROC AUC=0.7315454691492485\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.0075104583632683575\n",
            "remove:  6_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Energy  which is feature  62  out of  246\n",
            "New Test ROC AUC=0.7214924226352533\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.017563504877263547\n",
            "remove:  6_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Mean  which is feature  63  out of  246\n",
            "New Test ROC AUC=0.7263482847850649\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.012707642727451951\n",
            "remove:  6_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Std _Deviation  which is feature  64  out of  246\n",
            "New Test ROC AUC=0.7385501253925251\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.0005058021199917384\n",
            "remove:  6_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Energy  which is feature  65  out of  246\n",
            "New Test ROC AUC=0.7387550108512368\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.00030091666127995964\n",
            "remove:  6_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  6_Wavelet_Approximate_Entropy  which is feature  66  out of  246\n",
            "New Test ROC AUC=0.7334576757906761\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.005598251721840697\n",
            "remove:  6_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  6_Wavelet_Detailed_Entropy  which is feature  67  out of  246\n",
            "New Test ROC AUC=0.7283341234705946\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.010721804041922245\n",
            "remove:  6_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Mean  which is feature  68  out of  246\n",
            "New Test ROC AUC=0.7340747176457403\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.004981209866776526\n",
            "remove:  7_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Std_Deviation  which is feature  69  out of  246\n",
            "New Test ROC AUC=0.7290435773677537\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.010012350144763094\n",
            "remove:  7_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Energy  which is feature  70  out of  246\n",
            "New Test ROC AUC=0.7264859971916987\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.012569930320818101\n",
            "remove:  7_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Mean  which is feature  71  out of  246\n",
            "New Test ROC AUC=0.7264441927638136\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=-0.012611734748703252\n",
            "remove:  7_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Std _Deviation  which is feature  72  out of  246\n",
            "New Test ROC AUC=0.7559636013344998\n",
            "All features Test ROC AUC=0.7390559275125168\n",
            "Increase in ROC AUC=0.016907673821982994\n",
            "keep:  7_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Energy  which is feature  73  out of  246\n",
            "New Test ROC AUC=0.7589880774774349\n",
            "All features Test ROC AUC=0.7559636013344998\n",
            "Increase in ROC AUC=0.0030244761429351064\n",
            "keep:  7_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  7_Wavelet_Approximate_Entropy  which is feature  74  out of  246\n",
            "New Test ROC AUC=0.7582181709240244\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=-0.000769906553410471\n",
            "remove:  7_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  7_Wavelet_Detailed_Entropy  which is feature  75  out of  246\n",
            "New Test ROC AUC=0.7529730111656596\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=-0.006015066311775286\n",
            "remove:  7_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Mean  which is feature  76  out of  246\n",
            "New Test ROC AUC=0.7530929401727743\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=-0.005895137304660625\n",
            "remove:  8_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Std_Deviation  which is feature  77  out of  246\n",
            "New Test ROC AUC=0.7575786059913966\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=-0.0014094714860383162\n",
            "remove:  8_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Energy  which is feature  78  out of  246\n",
            "New Test ROC AUC=0.7591733762178097\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=0.0001852987403747619\n",
            "remove:  8_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Mean  which is feature  79  out of  246\n",
            "New Test ROC AUC=0.7503905772842308\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=-0.008597500193204155\n",
            "remove:  8_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Std _Deviation  which is feature  80  out of  246\n",
            "New Test ROC AUC=0.7683712495312679\n",
            "All features Test ROC AUC=0.7589880774774349\n",
            "Increase in ROC AUC=0.009383172053832989\n",
            "keep:  8_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Energy  which is feature  81  out of  246\n",
            "New Test ROC AUC=0.7695513245845133\n",
            "All features Test ROC AUC=0.7683712495312679\n",
            "Increase in ROC AUC=0.0011800750532453552\n",
            "keep:  8_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  8_Wavelet_Approximate_Entropy  which is feature  82  out of  246\n",
            "New Test ROC AUC=0.7665000397272976\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.0030512848572156193\n",
            "remove:  8_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  8_Wavelet_Detailed_Entropy  which is feature  83  out of  246\n",
            "New Test ROC AUC=0.7621511477753097\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.007400176809203574\n",
            "remove:  8_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Mean  which is feature  84  out of  246\n",
            "New Test ROC AUC=0.7400648062554118\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.02948651832910143\n",
            "remove:  9_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Std_Deviation  which is feature  85  out of  246\n",
            "New Test ROC AUC=0.7579704976704296\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.011580826914083686\n",
            "remove:  9_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Energy  which is feature  86  out of  246\n",
            "New Test ROC AUC=0.7599708053909896\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.009580519193523651\n",
            "remove:  9_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Mean  which is feature  87  out of  246\n",
            "New Test ROC AUC=0.7637370075762111\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.00581431700830215\n",
            "remove:  9_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Std _Deviation  which is feature  88  out of  246\n",
            "New Test ROC AUC=0.7698847745153286\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=0.0003334499308152994\n",
            "remove:  9_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Energy  which is feature  89  out of  246\n",
            "New Test ROC AUC=0.769550177626885\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-1.1469576282063798e-06\n",
            "remove:  9_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  9_Wavelet_Approximate_Entropy  which is feature  90  out of  246\n",
            "New Test ROC AUC=0.7696485716934441\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=9.724710893088684e-05\n",
            "remove:  9_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  9_Wavelet_Detailed_Entropy  which is feature  91  out of  246\n",
            "New Test ROC AUC=0.7618815248787635\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.007669799705749769\n",
            "remove:  9_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Mean  which is feature  92  out of  246\n",
            "New Test ROC AUC=0.7621881891903914\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.007363135394121811\n",
            "remove:  10_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Std_Deviation  which is feature  93  out of  246\n",
            "New Test ROC AUC=0.7614914023010038\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.00805992228350949\n",
            "remove:  10_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Energy  which is feature  94  out of  246\n",
            "New Test ROC AUC=0.7603059011388712\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=-0.009245423445642054\n",
            "remove:  10_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Mean  which is feature  95  out of  246\n",
            "New Test ROC AUC=0.7719149403597739\n",
            "All features Test ROC AUC=0.7695513245845133\n",
            "Increase in ROC AUC=0.0023636157752606746\n",
            "keep:  10_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Std _Deviation  which is feature  96  out of  246\n",
            "New Test ROC AUC=0.7694868178041493\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.0024281225556246344\n",
            "remove:  10_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Energy  which is feature  97  out of  246\n",
            "New Test ROC AUC=0.7701146698371648\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.0018002705226091154\n",
            "remove:  10_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  10_Wavelet_Approximate_Entropy  which is feature  98  out of  246\n",
            "New Test ROC AUC=0.7674732676174647\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.004441672742309222\n",
            "remove:  10_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  10_Wavelet_Detailed_Entropy  which is feature  99  out of  246\n",
            "New Test ROC AUC=0.7600088956434702\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.011906044716303743\n",
            "remove:  10_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Mean  which is feature  100  out of  246\n",
            "New Test ROC AUC=0.7564839299902666\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.015431010369507336\n",
            "remove:  11_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Std_Deviation  which is feature  101  out of  246\n",
            "New Test ROC AUC=0.7581423445689169\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.01377259579085699\n",
            "remove:  11_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Energy  which is feature  102  out of  246\n",
            "New Test ROC AUC=0.7695102608168234\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.0024046795429505385\n",
            "remove:  11_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Mean  which is feature  103  out of  246\n",
            "New Test ROC AUC=0.7655661599632853\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.0063487803964886735\n",
            "remove:  11_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Std _Deviation  which is feature  104  out of  246\n",
            "New Test ROC AUC=0.7705405011360189\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.0013744392237550196\n",
            "remove:  11_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Energy  which is feature  105  out of  246\n",
            "New Test ROC AUC=0.7715274937160541\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.0003874466437198132\n",
            "remove:  11_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  11_Wavelet_Approximate_Entropy  which is feature  106  out of  246\n",
            "New Test ROC AUC=0.7691898841877989\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.002725056171975049\n",
            "remove:  11_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  11_Wavelet_Detailed_Entropy  which is feature  107  out of  246\n",
            "New Test ROC AUC=0.7567023527645406\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.015212587595233362\n",
            "remove:  11_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Mean  which is feature  108  out of  246\n",
            "New Test ROC AUC=0.7594165271859822\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.012498413173791745\n",
            "remove:  12_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Std_Deviation  which is feature  109  out of  246\n",
            "New Test ROC AUC=0.7625744904683308\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.009340449891443114\n",
            "remove:  12_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Energy  which is feature  110  out of  246\n",
            "New Test ROC AUC=0.755596089416473\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.016318850943300878\n",
            "remove:  12_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Mean  which is feature  111  out of  246\n",
            "New Test ROC AUC=0.7599901313626629\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=-0.011924808997111014\n",
            "remove:  12_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Std _Deviation  which is feature  112  out of  246\n",
            "New Test ROC AUC=0.7764148826851994\n",
            "All features Test ROC AUC=0.7719149403597739\n",
            "Increase in ROC AUC=0.004499942325425499\n",
            "keep:  12_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Energy  which is feature  113  out of  246\n",
            "New Test ROC AUC=0.7840241371836398\n",
            "All features Test ROC AUC=0.7764148826851994\n",
            "Increase in ROC AUC=0.00760925449844041\n",
            "keep:  12_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  12_Wavelet_Approximate_Entropy  which is feature  114  out of  246\n",
            "New Test ROC AUC=0.77782769868984\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=-0.006196438493799805\n",
            "remove:  12_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  12_Wavelet_Detailed_Entropy  which is feature  115  out of  246\n",
            "New Test ROC AUC=0.7778696917916988\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=-0.006154445391940988\n",
            "remove:  12_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Mean  which is feature  116  out of  246\n",
            "New Test ROC AUC=0.7768578321958274\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=-0.007166304987812455\n",
            "remove:  13_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Std_Deviation  which is feature  117  out of  246\n",
            "New Test ROC AUC=0.7760969257635957\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=-0.007927211420044156\n",
            "remove:  13_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Energy  which is feature  118  out of  246\n",
            "New Test ROC AUC=0.7740963181606736\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=-0.009927819022966244\n",
            "remove:  13_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Mean  which is feature  119  out of  246\n",
            "New Test ROC AUC=0.778747281694575\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=-0.005276855489064869\n",
            "remove:  13_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Std _Deviation  which is feature  120  out of  246\n",
            "New Test ROC AUC=0.7855749273553319\n",
            "All features Test ROC AUC=0.7840241371836398\n",
            "Increase in ROC AUC=0.001550790171692018\n",
            "keep:  13_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Energy  which is feature  121  out of  246\n",
            "New Test ROC AUC=0.7847925284062331\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.000782398949098706\n",
            "remove:  13_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  13_Wavelet_Approximate_Entropy  which is feature  122  out of  246\n",
            "New Test ROC AUC=0.7815084042103585\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.004066523144973355\n",
            "remove:  13_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  13_Wavelet_Detailed_Entropy  which is feature  123  out of  246\n",
            "New Test ROC AUC=0.7836776912950203\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.001897236060311558\n",
            "remove:  13_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Mean  which is feature  124  out of  246\n",
            "New Test ROC AUC=0.7839132053461999\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.0016617220091319496\n",
            "remove:  14_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Std_Deviation  which is feature  125  out of  246\n",
            "New Test ROC AUC=0.7789685822454202\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.006606345109911649\n",
            "remove:  14_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Energy  which is feature  126  out of  246\n",
            "New Test ROC AUC=0.7821324557286844\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.0034424716266474586\n",
            "remove:  14_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Mean  which is feature  127  out of  246\n",
            "New Test ROC AUC=0.7808835543546392\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=-0.004691373000692667\n",
            "remove:  14_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Std _Deviation  which is feature  128  out of  246\n",
            "New Test ROC AUC=0.7896503605189379\n",
            "All features Test ROC AUC=0.7855749273553319\n",
            "Increase in ROC AUC=0.004075433163606035\n",
            "keep:  14_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Energy  which is feature  129  out of  246\n",
            "New Test ROC AUC=0.7864632163638483\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=-0.003187144155089583\n",
            "remove:  14_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  14_Wavelet_Approximate_Entropy  which is feature  130  out of  246\n",
            "New Test ROC AUC=0.788682171907252\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=-0.0009681886116859051\n",
            "remove:  14_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  14_Wavelet_Detailed_Entropy  which is feature  131  out of  246\n",
            "New Test ROC AUC=0.7832457751900533\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=-0.006404585328884549\n",
            "remove:  14_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Mean  which is feature  132  out of  246\n",
            "New Test ROC AUC=0.7851601548579539\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=-0.004490205660984015\n",
            "remove:  15_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Std_Deviation  which is feature  133  out of  246\n",
            "New Test ROC AUC=0.7877206691085225\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=-0.001929691410415435\n",
            "remove:  15_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Energy  which is feature  134  out of  246\n",
            "New Test ROC AUC=0.7820219701557314\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=-0.007628390363206505\n",
            "remove:  15_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Mean  which is feature  135  out of  246\n",
            "New Test ROC AUC=0.7932907514954268\n",
            "All features Test ROC AUC=0.7896503605189379\n",
            "Increase in ROC AUC=0.0036403909764889386\n",
            "keep:  15_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Std _Deviation  which is feature  136  out of  246\n",
            "New Test ROC AUC=0.7851377987206342\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008152952774792666\n",
            "remove:  15_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Energy  which is feature  137  out of  246\n",
            "New Test ROC AUC=0.7852541088656222\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008036642629804636\n",
            "remove:  15_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  15_Wavelet_Approximate_Entropy  which is feature  138  out of  246\n",
            "New Test ROC AUC=0.7868852376616692\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.006405513833757648\n",
            "remove:  15_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  15_Wavelet_Detailed_Entropy  which is feature  139  out of  246\n",
            "New Test ROC AUC=0.7849184835768889\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008372267918537957\n",
            "remove:  15_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Mean  which is feature  140  out of  246\n",
            "New Test ROC AUC=0.787831240681803\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.005459510813623836\n",
            "remove:  16_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Std_Deviation  which is feature  141  out of  246\n",
            "New Test ROC AUC=0.7882748744209018\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.005015877074524999\n",
            "remove:  16_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Energy  which is feature  142  out of  246\n",
            "New Test ROC AUC=0.7833451365329767\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.009945614962450167\n",
            "remove:  16_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Mean  which is feature  143  out of  246\n",
            "New Test ROC AUC=0.7835080894867497\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.009782662008677168\n",
            "remove:  16_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Std _Deviation  which is feature  144  out of  246\n",
            "New Test ROC AUC=0.7867407385359689\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.006550012959457896\n",
            "remove:  16_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Energy  which is feature  145  out of  246\n",
            "New Test ROC AUC=0.7868081340294854\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.006482617465941476\n",
            "remove:  16_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  16_Wavelet_Approximate_Entropy  which is feature  146  out of  246\n",
            "New Test ROC AUC=0.7829152948426202\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.010375456652806592\n",
            "remove:  16_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  16_Wavelet_Detailed_Entropy  which is feature  147  out of  246\n",
            "New Test ROC AUC=0.7850861901928461\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008204561302580737\n",
            "remove:  16_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Mean  which is feature  148  out of  246\n",
            "New Test ROC AUC=0.7751702243783348\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.018120527117092\n",
            "remove:  17_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Std_Deviation  which is feature  149  out of  246\n",
            "New Test ROC AUC=0.7888647502587418\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.004426001236685018\n",
            "remove:  17_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Energy  which is feature  150  out of  246\n",
            "New Test ROC AUC=0.7861049138846712\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.007185837610755641\n",
            "remove:  17_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Mean  which is feature  151  out of  246\n",
            "New Test ROC AUC=0.7788392106413266\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.014451540854100275\n",
            "remove:  17_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Std _Deviation  which is feature  152  out of  246\n",
            "New Test ROC AUC=0.792443748839532\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.0008470026558948796\n",
            "remove:  17_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Energy  which is feature  153  out of  246\n",
            "New Test ROC AUC=0.7924343915727343\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.0008563599226925067\n",
            "remove:  17_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  17_Wavelet_Approximate_Entropy  which is feature  154  out of  246\n",
            "New Test ROC AUC=0.7862699448359772\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.007020806659449597\n",
            "remove:  17_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  17_Wavelet_Detailed_Entropy  which is feature  155  out of  246\n",
            "New Test ROC AUC=0.7873090119647735\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.005981739530653285\n",
            "remove:  17_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Mean  which is feature  156  out of  246\n",
            "New Test ROC AUC=0.7825617355225285\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.010729015972898326\n",
            "remove:  18_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Std_Deviation  which is feature  157  out of  246\n",
            "New Test ROC AUC=0.7794418275494673\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.01384892394595949\n",
            "remove:  18_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Energy  which is feature  158  out of  246\n",
            "New Test ROC AUC=0.7889756436636486\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.004315107831778242\n",
            "remove:  18_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Mean  which is feature  159  out of  246\n",
            "New Test ROC AUC=0.7758350004249287\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.017455751070498127\n",
            "remove:  18_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Std _Deviation  which is feature  160  out of  246\n",
            "New Test ROC AUC=0.7880395969357924\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.00525115455963443\n",
            "remove:  18_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Energy  which is feature  161  out of  246\n",
            "New Test ROC AUC=0.78815354158429\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.005137209911136864\n",
            "remove:  18_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  18_Wavelet_Approximate_Entropy  which is feature  162  out of  246\n",
            "New Test ROC AUC=0.7884024311368834\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.004888320358543474\n",
            "remove:  18_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  18_Wavelet_Detailed_Entropy  which is feature  163  out of  246\n",
            "New Test ROC AUC=0.7843501078980913\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008940643597335485\n",
            "remove:  18_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Mean  which is feature  164  out of  246\n",
            "New Test ROC AUC=0.7794036669867707\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.013887084508656167\n",
            "remove:  19_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Std_Deviation  which is feature  165  out of  246\n",
            "New Test ROC AUC=0.7840782620225085\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.009212489472918306\n",
            "remove:  19_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Energy  which is feature  166  out of  246\n",
            "New Test ROC AUC=0.7800512166051411\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.013239534890285731\n",
            "remove:  19_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Mean  which is feature  167  out of  246\n",
            "New Test ROC AUC=0.7754912235561173\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.0177995279393095\n",
            "remove:  19_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Std _Deviation  which is feature  168  out of  246\n",
            "New Test ROC AUC=0.7787469161069722\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.014543835388454607\n",
            "remove:  19_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Energy  which is feature  169  out of  246\n",
            "New Test ROC AUC=0.7789444558359251\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.01434629565950174\n",
            "remove:  19_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  19_Wavelet_Approximate_Entropy  which is feature  170  out of  246\n",
            "New Test ROC AUC=0.7830946172584825\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.01019613423694432\n",
            "remove:  19_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  19_Wavelet_Detailed_Entropy  which is feature  171  out of  246\n",
            "New Test ROC AUC=0.7837918821447808\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.009498869350646033\n",
            "remove:  19_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Mean  which is feature  172  out of  246\n",
            "New Test ROC AUC=0.7741194783612361\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.01917127313419076\n",
            "remove:  20_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Std_Deviation  which is feature  173  out of  246\n",
            "New Test ROC AUC=0.781223961359827\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.012066790135599859\n",
            "remove:  20_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Energy  which is feature  174  out of  246\n",
            "New Test ROC AUC=0.7779789481303458\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.015311803365080978\n",
            "remove:  20_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Mean  which is feature  175  out of  246\n",
            "New Test ROC AUC=0.7826508356469984\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.010639915848428383\n",
            "remove:  20_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Std _Deviation  which is feature  176  out of  246\n",
            "New Test ROC AUC=0.7874466879162945\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.005844063579132364\n",
            "remove:  20_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Energy  which is feature  177  out of  246\n",
            "New Test ROC AUC=0.7865143614701435\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.0067763900252832965\n",
            "remove:  20_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  20_Wavelet_Approximate_Entropy  which is feature  178  out of  246\n",
            "New Test ROC AUC=0.7861370016706937\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.007153749824733158\n",
            "remove:  20_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  20_Wavelet_Detailed_Entropy  which is feature  179  out of  246\n",
            "New Test ROC AUC=0.785180807855034\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008109943640392858\n",
            "remove:  20_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Mean  which is feature  180  out of  246\n",
            "New Test ROC AUC=0.785319274260459\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.007971477234967828\n",
            "remove:  21_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Std_Deviation  which is feature  181  out of  246\n",
            "New Test ROC AUC=0.782830925119441\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.010459826375985859\n",
            "remove:  21_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Energy  which is feature  182  out of  246\n",
            "New Test ROC AUC=0.772694644944043\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.020596106551383775\n",
            "remove:  21_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Mean  which is feature  183  out of  246\n",
            "New Test ROC AUC=0.7745320715397931\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.01875867995563374\n",
            "remove:  21_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Std _Deviation  which is feature  184  out of  246\n",
            "New Test ROC AUC=0.7882868190065698\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.005003932488856999\n",
            "remove:  21_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Energy  which is feature  185  out of  246\n",
            "New Test ROC AUC=0.7883459107384043\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.004944840757022506\n",
            "remove:  21_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  21_Wavelet_Approximate_Entropy  which is feature  186  out of  246\n",
            "New Test ROC AUC=0.7895224518924364\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.0037682996029904103\n",
            "remove:  21_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  21_Wavelet_Detailed_Entropy  which is feature  187  out of  246\n",
            "New Test ROC AUC=0.7806563393822396\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.012634412113187277\n",
            "remove:  21_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Mean  which is feature  188  out of  246\n",
            "New Test ROC AUC=0.777474563367967\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.015816188127459774\n",
            "remove:  22_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Std_Deviation  which is feature  189  out of  246\n",
            "New Test ROC AUC=0.7808872622680071\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.01240348922741974\n",
            "remove:  22_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Energy  which is feature  190  out of  246\n",
            "New Test ROC AUC=0.7850098342852925\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.008280917210134353\n",
            "remove:  22_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Mean  which is feature  191  out of  246\n",
            "New Test ROC AUC=0.7772206307831667\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=-0.016070120712260105\n",
            "remove:  22_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Std _Deviation  which is feature  192  out of  246\n",
            "New Test ROC AUC=0.796833425810942\n",
            "All features Test ROC AUC=0.7932907514954268\n",
            "Increase in ROC AUC=0.0035426743155151375\n",
            "keep:  22_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Energy  which is feature  193  out of  246\n",
            "New Test ROC AUC=0.7925619008688881\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.00427152494205385\n",
            "remove:  22_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  22_Wavelet_Approximate_Entropy  which is feature  194  out of  246\n",
            "New Test ROC AUC=0.7926508946758134\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.004182531135128564\n",
            "remove:  22_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  22_Wavelet_Detailed_Entropy  which is feature  195  out of  246\n",
            "New Test ROC AUC=0.7894737453799642\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.007359680430977789\n",
            "remove:  22_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Mean  which is feature  196  out of  246\n",
            "New Test ROC AUC=0.7839227651026623\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.01291066070827962\n",
            "remove:  23_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Std_Deviation  which is feature  197  out of  246\n",
            "New Test ROC AUC=0.7887644582186972\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.008068967592244736\n",
            "remove:  23_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Energy  which is feature  198  out of  246\n",
            "New Test ROC AUC=0.7842001198101229\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.012633306000819022\n",
            "remove:  23_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Mean  which is feature  199  out of  246\n",
            "New Test ROC AUC=0.782588466241275\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.014244959569666982\n",
            "remove:  23_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Std _Deviation  which is feature  200  out of  246\n",
            "New Test ROC AUC=0.7910486667105616\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.005784759100380343\n",
            "remove:  23_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Energy  which is feature  201  out of  246\n",
            "New Test ROC AUC=0.7908347823934488\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.005998643417493121\n",
            "remove:  23_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  23_Wavelet_Approximate_Entropy  which is feature  202  out of  246\n",
            "New Test ROC AUC=0.7936244328557988\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0032089929551432084\n",
            "remove:  23_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  23_Wavelet_Detailed_Entropy  which is feature  203  out of  246\n",
            "New Test ROC AUC=0.7806903245565319\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.016143101254410097\n",
            "remove:  23_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Mean  which is feature  204  out of  246\n",
            "New Test ROC AUC=0.7804696354445297\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.016363790366412223\n",
            "remove:  24_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Std_Deviation  which is feature  205  out of  246\n",
            "New Test ROC AUC=0.7860577434538332\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.010775682357108796\n",
            "remove:  24_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Energy  which is feature  206  out of  246\n",
            "New Test ROC AUC=0.7866717911342173\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.010161634676724707\n",
            "remove:  24_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Mean  which is feature  207  out of  246\n",
            "New Test ROC AUC=0.7875476022756133\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.009285823535328697\n",
            "remove:  24_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Std _Deviation  which is feature  208  out of  246\n",
            "New Test ROC AUC=0.7843774495253202\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.012455976285621784\n",
            "remove:  24_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Energy  which is feature  209  out of  246\n",
            "New Test ROC AUC=0.7857556033271798\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.011077822483762212\n",
            "remove:  24_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  24_Wavelet_Approximate_Entropy  which is feature  210  out of  246\n",
            "New Test ROC AUC=0.7934964505462195\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0033369752647224304\n",
            "remove:  24_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  24_Wavelet_Detailed_Entropy  which is feature  211  out of  246\n",
            "New Test ROC AUC=0.7852927276763948\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.011540698134547167\n",
            "remove:  24_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Mean  which is feature  212  out of  246\n",
            "New Test ROC AUC=0.787402720531553\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.00943070527938894\n",
            "remove:  25_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Std_Deviation  which is feature  213  out of  246\n",
            "New Test ROC AUC=0.7959092027858558\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0009242230250862127\n",
            "remove:  25_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Energy  which is feature  214  out of  246\n",
            "New Test ROC AUC=0.783431636810736\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.013401789000205944\n",
            "remove:  25_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Mean  which is feature  215  out of  246\n",
            "New Test ROC AUC=0.7904792484436299\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0063541773673120305\n",
            "remove:  25_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Std _Deviation  which is feature  216  out of  246\n",
            "New Test ROC AUC=0.7952595333519616\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0015738924589803194\n",
            "remove:  25_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Energy  which is feature  217  out of  246\n",
            "New Test ROC AUC=0.7944642435144808\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.002369182296461214\n",
            "remove:  25_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  25_Wavelet_Approximate_Entropy  which is feature  218  out of  246\n",
            "New Test ROC AUC=0.7954739122305358\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.001359513580406202\n",
            "remove:  25_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  25_Wavelet_Detailed_Entropy  which is feature  219  out of  246\n",
            "New Test ROC AUC=0.7808810994791913\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0159523263317507\n",
            "remove:  25_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Mean  which is feature  220  out of  246\n",
            "New Test ROC AUC=0.7902734806297965\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0065599451811454434\n",
            "remove:  26_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Std_Deviation  which is feature  221  out of  246\n",
            "New Test ROC AUC=0.7830061886066312\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.013827237204310783\n",
            "remove:  26_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Energy  which is feature  222  out of  246\n",
            "New Test ROC AUC=0.7875305302882807\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.00930289552266128\n",
            "remove:  26_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Mean  which is feature  223  out of  246\n",
            "New Test ROC AUC=0.7917116850075231\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.005121740803418873\n",
            "remove:  26_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Std _Deviation  which is feature  224  out of  246\n",
            "New Test ROC AUC=0.7891696617274142\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.007663764083527758\n",
            "remove:  26_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Energy  which is feature  225  out of  246\n",
            "New Test ROC AUC=0.7894323775257532\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.007401048285188727\n",
            "remove:  26_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  26_Wavelet_Approximate_Entropy  which is feature  226  out of  246\n",
            "New Test ROC AUC=0.7911816878707315\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.005651737940210455\n",
            "remove:  26_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  26_Wavelet_Detailed_Entropy  which is feature  227  out of  246\n",
            "New Test ROC AUC=0.7806868212097172\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.016146604601224746\n",
            "remove:  26_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Mean  which is feature  228  out of  246\n",
            "New Test ROC AUC=0.7857573241416611\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.011076101669280858\n",
            "remove:  27_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Std_Deviation  which is feature  229  out of  246\n",
            "New Test ROC AUC=0.7880826931757373\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.00875073263520465\n",
            "remove:  27_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Energy  which is feature  230  out of  246\n",
            "New Test ROC AUC=0.7826209130578133\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.014212512753128625\n",
            "remove:  27_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Mean  which is feature  231  out of  246\n",
            "New Test ROC AUC=0.7859349573384466\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.010898468472495315\n",
            "remove:  27_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Std _Deviation  which is feature  232  out of  246\n",
            "New Test ROC AUC=0.7958072293022801\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0010261965086618696\n",
            "remove:  27_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Energy  which is feature  233  out of  246\n",
            "New Test ROC AUC=0.7956980275425375\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.001135398268404475\n",
            "remove:  27_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  27_Wavelet_Approximate_Entropy  which is feature  234  out of  246\n",
            "New Test ROC AUC=0.7975998364537726\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=0.0007664106428306727\n",
            "remove:  27_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  27_Wavelet_Detailed_Entropy  which is feature  235  out of  246\n",
            "New Test ROC AUC=0.781669010867163\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.015164414943778959\n",
            "remove:  27_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Mean  which is feature  236  out of  246\n",
            "New Test ROC AUC=0.7806690540303173\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.016164371780624687\n",
            "remove:  28_Wavelet_Approximate_Mean\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Std_Deviation  which is feature  237  out of  246\n",
            "New Test ROC AUC=0.7939364611987272\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0028969646122147807\n",
            "remove:  28_Wavelet_Approximate_Std_Deviation\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Energy  which is feature  238  out of  246\n",
            "New Test ROC AUC=0.7899403204200274\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.006893105390914589\n",
            "remove:  28_Wavelet_Approximate_Energy\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Mean  which is feature  239  out of  246\n",
            "New Test ROC AUC=0.7769235214960714\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.019909904314870608\n",
            "remove:  28_Wavelet_Detailed_Mean\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Std _Deviation  which is feature  240  out of  246\n",
            "New Test ROC AUC=0.795625081265312\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0012083445456300135\n",
            "remove:  28_Wavelet_Detailed_Std _Deviation\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Energy  which is feature  241  out of  246\n",
            "New Test ROC AUC=0.7963102759138577\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0005231498970842408\n",
            "remove:  28_Wavelet_Detailed_Energy\n",
            "\n",
            "testing feature:  28_Wavelet_Approximate_Entropy  which is feature  242  out of  246\n",
            "New Test ROC AUC=0.7957916382730728\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.0010417875378692099\n",
            "remove:  28_Wavelet_Approximate_Entropy\n",
            "\n",
            "testing feature:  28_Wavelet_Detailed_Entropy  which is feature  243  out of  246\n",
            "New Test ROC AUC=0.7888264939921917\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.00800693181875023\n",
            "remove:  28_Wavelet_Detailed_Entropy\n",
            "\n",
            "testing feature:  Hurst_Exponent  which is feature  244  out of  246\n",
            "New Test ROC AUC=0.7827185002838425\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=-0.014114925527099476\n",
            "remove:  Hurst_Exponent\n",
            "\n",
            "testing feature:  Permutation_Entropy  which is feature  245  out of  246\n",
            "New Test ROC AUC=0.7976953763530877\n",
            "All features Test ROC AUC=0.796833425810942\n",
            "Increase in ROC AUC=0.0008619505421457729\n",
            "remove:  Permutation_Entropy\n",
            "DONE!!\n",
            "total features to keep:  36\n"
          ]
        }
      ],
      "source": [
        "features_to_keep = recursive_feature_addition_rf(X_train=X_train,\n",
        "                                                        y_train=y_train,\n",
        "                                                        X_test=X_test,\n",
        "                                                        y_test=y_test,\n",
        "                                                        tol=0.001)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTsKJprtu5lS",
        "outputId": "ee4bc344-bf88-466e-b78f-e395b483f2ae"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Coeffiecient_of_Variation',\n",
              " 'Mean_of_Vertex_to_Vertex_Slope',\n",
              " 'Variance_of_Vertex_to_Vertex_Slope',\n",
              " 'Hjorth_Activity',\n",
              " 'Hjorth_Mobility',\n",
              " 'Hjorth_Complexity',\n",
              " 'Kurtosis',\n",
              " '2nd_Difference_Mean',\n",
              " '2nd_Difference_Max',\n",
              " '1st_Difference_Mean',\n",
              " 'FFT_Delta_MaxPower',\n",
              " 'FFT_Theta_MaxPower',\n",
              " 'FFT_Alpha_MaxPower',\n",
              " 'FFT_Beta_MaxPower',\n",
              " '1_Wavelet_Detailed_Std_Deviation',\n",
              " '1_Wavelet_Detailed_Energy',\n",
              " '2_Wavelet_Detailed_Std _Deviation',\n",
              " '2_Wavelet_Detailed_Energy',\n",
              " '3_Wavelet_Approximate_Mean',\n",
              " '3_Wavelet_Detailed_Std _Deviation',\n",
              " '3_Wavelet_Detailed_Energy',\n",
              " '4_Wavelet_Detailed_Std _Deviation',\n",
              " '5_Wavelet_Detailed_Std _Deviation',\n",
              " '5_Wavelet_Detailed_Energy',\n",
              " '5_Wavelet_Approximate_Entropy',\n",
              " '7_Wavelet_Detailed_Std _Deviation',\n",
              " '7_Wavelet_Detailed_Energy',\n",
              " '8_Wavelet_Detailed_Std _Deviation',\n",
              " '8_Wavelet_Detailed_Energy',\n",
              " '10_Wavelet_Detailed_Mean',\n",
              " '12_Wavelet_Detailed_Std _Deviation',\n",
              " '12_Wavelet_Detailed_Energy',\n",
              " '13_Wavelet_Detailed_Std _Deviation',\n",
              " '14_Wavelet_Detailed_Std _Deviation',\n",
              " '15_Wavelet_Detailed_Mean',\n",
              " '22_Wavelet_Detailed_Std _Deviation']"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "features_to_keep"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QZIsZL9xpc81"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mAFjxg8gpdBG"
      },
      "outputs": [],
      "source": [
        "from sklearn.ensemble import RandomForestClassifier #, RandomForestRegressor\n",
        "from sklearn.metrics import roc_auc_score #, mean_squared_error\n",
        "\n",
        "# 2018.12.02 Created by Eamon.Zhang\n",
        "\n",
        "\n",
        "def recursive_feature_elimination_rf(X_train,y_train,X_test,y_test,\n",
        "                                     tol=0.001,max_depth=None,\n",
        "                                     class_weight=None,\n",
        "                                     top_n=15,n_estimators=50,random_state=0):\n",
        "    \n",
        "   \n",
        "    features_to_remove = []\n",
        "    count = 1\n",
        "    # initial model using all the features\n",
        "    model_all_features = RandomForestClassifier(n_estimators=n_estimators,max_depth=max_depth,\n",
        "                                    random_state=random_state,class_weight=class_weight,\n",
        "                                    n_jobs=-1)\n",
        "    model_all_features.fit(X_train, y_train)\n",
        "    y_pred_test = model_all_features.predict_proba(X_test)\n",
        "    auc_score_all = roc_auc_score(y_test, y_pred_test,multi_class = 'ovr', average = 'weighted')\n",
        "    \n",
        "    for feature in X_train.columns:\n",
        "        print()\n",
        "        print('testing feature: ', feature, ' which is feature ', count,\n",
        "          ' out of ', len(X_train.columns))\n",
        "        count += 1\n",
        "        model = RandomForestClassifier(n_estimators=n_estimators,max_depth=max_depth,\n",
        "                                    random_state=random_state,class_weight=class_weight,\n",
        "                                    n_jobs=-1)\n",
        "        \n",
        "        # fit model with all variables minus the removed features\n",
        "        # and the feature to be evaluated\n",
        "        model.fit(X_train.drop(features_to_remove + [feature], axis=1), y_train)\n",
        "        y_pred_test = model.predict_proba(\n",
        "                    X_test.drop(features_to_remove + [feature], axis=1))   \n",
        "        auc_score_int = roc_auc_score(y_test, y_pred_test,multi_class = 'ovr', average = 'weighted')\n",
        "        print('New Test ROC AUC={}'.format((auc_score_int)))\n",
        "    \n",
        "        # print the original roc-auc with all the features\n",
        "        print('All features Test ROC AUC={}'.format((auc_score_all)))\n",
        "    \n",
        "        # determine the drop in the roc-auc\n",
        "        diff_auc = auc_score_all - auc_score_int\n",
        "    \n",
        "        # compare the drop in roc-auc with the tolerance\n",
        "        if diff_auc >= tol:\n",
        "            print('Drop in ROC AUC={}'.format(diff_auc))\n",
        "            print('keep: ', feature)\n",
        "            \n",
        "        else:\n",
        "            print('Drop in ROC AUC={}'.format(diff_auc))\n",
        "            print('remove: ', feature)\n",
        "            \n",
        "            # if the drop in the roc is small and we remove the\n",
        "            # feature, we need to set the new roc to the one based on\n",
        "            # the remaining features\n",
        "            auc_score_all = auc_score_int\n",
        "            \n",
        "            # and append the feature to remove to the list\n",
        "            features_to_remove.append(feature)\n",
        "    print('DONE!!')\n",
        "    print('total features to remove: ', len(features_to_remove))  \n",
        "    features_to_keep = [x for x in X_train.columns if x not in features_to_remove]\n",
        "    print('total features to keep: ', len(features_to_keep))\n",
        "    \n",
        "    return features_to_keep\n",
        "\n",
        "\n",
        "def recursive_feature_addition_rf(X_train,y_train,X_test,y_test,\n",
        "                                     tol=0.001,max_depth=None,\n",
        "                                     class_weight=None,\n",
        "                                     top_n=15,n_estimators=50,random_state=0):\n",
        "    \n",
        "   \n",
        "    features_to_keep = [X_train.columns[0]]\n",
        "    count = 1\n",
        "    # initial model using only one feature\n",
        "    model_one_feature = RandomForestClassifier(n_estimators=n_estimators,max_depth=max_depth,\n",
        "                                    random_state=random_state,class_weight=class_weight,\n",
        "                                    n_jobs=-1)\n",
        "    model_one_feature.fit(X_train[[X_train.columns[0]]], y_train)\n",
        "    y_pred_test = model_one_feature.predict_proba(X_test[[X_train.columns[0]]]) \n",
        "    auc_score_all = roc_auc_score(y_test, y_pred_test,multi_class = 'ovr', average = 'weighted')\n",
        "    \n",
        "    for feature in X_train.columns[1:]:\n",
        "        print()\n",
        "        print('testing feature: ', feature, ' which is feature ', count,\n",
        "          ' out of ', len(X_train.columns))\n",
        "        count += 1\n",
        "        model = RandomForestClassifier(n_estimators=n_estimators,max_depth=max_depth,\n",
        "                                    random_state=random_state,class_weight=class_weight,\n",
        "                                    n_jobs=-1)\n",
        "        \n",
        "        # fit model with  the selected features\n",
        "        # and the feature to be evaluated\n",
        "        model.fit(X_train[features_to_keep + [feature]], y_train)\n",
        "        y_pred_test = model.predict_proba(\n",
        "                    X_test[features_to_keep + [feature]])  \n",
        "        auc_score_int = roc_auc_score(y_test, y_pred_test )\n",
        "        print('New Test ROC AUC={}'.format((auc_score_int)))\n",
        "    \n",
        "        # print the original roc-auc with all the features\n",
        "        print('All features Test ROC AUC={}'.format((auc_score_all)))\n",
        "    \n",
        "        # determine the drop in the roc-auc\n",
        "        diff_auc = auc_score_int - auc_score_all\n",
        "    \n",
        "        # compare the drop in roc-auc with the tolerance\n",
        "        if diff_auc >= tol:\n",
        "            # if the increase in the roc is bigger than the threshold\n",
        "            # we keep the feature and re-adjust the roc-auc to the new value\n",
        "            # considering the added feature\n",
        "            print('Increase in ROC AUC={}'.format(diff_auc))\n",
        "            print('keep: ', feature)\n",
        "            auc_score_all = auc_score_int\n",
        "            features_to_keep.append(feature)\n",
        "        else:\n",
        "            print('Increase in ROC AUC={}'.format(diff_auc))\n",
        "            print('remove: ', feature)          \n",
        "\n",
        "    print('DONE!!')\n",
        "    print('total features to keep: ', len(features_to_keep))  \n",
        "   \n",
        "    return features_to_keep"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yUgu0VjUwPgT"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}