feature_selection/feature_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nuclio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%nuclio: setting kind to 'job'\n",
      "%nuclio: setting spec.image to 'mlrun/ml-models'\n"
     ]
    }
   ],
   "source": [
    "%nuclio config kind = \"job\"\n",
    "%nuclio config spec.image = \"mlrun/ml-models\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: start-code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import os\n",
    "import json\n",
    "\n",
    "# Feature selection strategies\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "# Model based feature selection\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# Scale feature scores\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "# SKLearn estimators list\n",
    "from sklearn.utils import all_estimators\n",
    "\n",
    "# MLRun utils\n",
    "from mlrun.mlutils import create_class, gcf_clear\n",
    "from mlrun.artifacts import PlotArtifact"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_values_on_bars(axs, h_v=\"v\", space=0.4):\n",
    "    def _show_on_single_plot(ax):\n",
    "        if h_v == \"v\":\n",
    "            for p in ax.patches:\n",
    "                _x = p.get_x() + p.get_width() / 2\n",
    "                _y = p.get_y() + p.get_height()\n",
    "                value = int(p.get_height())\n",
    "                ax.text(_x, _y, value, ha=\"center\") \n",
    "        elif h_v == \"h\":\n",
    "            for p in ax.patches:\n",
    "                _x = p.get_x() + p.get_width() + float(space)\n",
    "                _y = p.get_y() + p.get_height()\n",
    "                value = int(p.get_width())\n",
    "                ax.text(_x, _y, value, ha=\"left\")\n",
    "\n",
    "    if isinstance(axs, np.ndarray):\n",
    "        for idx, ax in np.ndenumerate(axs):\n",
    "            _show_on_single_plot(ax)\n",
    "    else:\n",
    "        _show_on_single_plot(axs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_stat(context,\n",
    "              stat_name,\n",
    "              stat_df):\n",
    "    gcf_clear(plt)\n",
    "        \n",
    "    # Add chart\n",
    "    ax = plt.axes()\n",
    "    stat_chart = sns.barplot(x=stat_name, \n",
    "                            y='index', \n",
    "                            data=stat_df.sort_values(stat_name, ascending=False).reset_index(), \n",
    "                            ax=ax)\n",
    "    plt.tight_layout()\n",
    "\n",
    "    for p in stat_chart.patches:\n",
    "        width = p.get_width()\n",
    "        plt.text(5+p.get_width(), p.get_y()+0.55*p.get_height(),\n",
    "                 '{:1.2f}'.format(width),\n",
    "                 ha='center', va='center')\n",
    "\n",
    "    context.log_artifact(PlotArtifact(f'{stat_name}', body=plt.gcf()), \n",
    "                         local_path=os.path.join('plots', 'feature_selection', f'{stat_name}.html'))\n",
    "    gcf_clear(plt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_selection(context,\n",
    "                      df_artifact,\n",
    "                      k=2,\n",
    "                      min_votes=0.5,\n",
    "                      label_column: str = 'Y',\n",
    "                      stat_filters = ['f_classif', 'mutual_info_classif', 'chi2', 'f_regression'],\n",
    "                      model_filters = {'LinearSVC': 'LinearSVC', \n",
    "                                       'LogisticRegression': 'LogisticRegression', \n",
    "                                       'ExtraTreesClassifier': 'ExtraTreesClassifier'},\n",
    "                      max_scaled_scores = True):\n",
    "    \"\"\"Applies selected feature selection statistical functions \n",
    "    or models on our 'df_artifact'.\n",
    "    \n",
    "    Each statistical function or model will vote for it's best K selected features.\n",
    "    If a feature has >= 'min_votes' votes, it will be selected.\n",
    "\n",
    "    :param context:           the function context\n",
    "    :param k:                 number of top features to select from each statistical\n",
    "                              function or model\n",
    "    :param min_votes:         minimal number of votes (from a model or by statistical\n",
    "                              function) needed for a feature to be selected.\n",
    "                              Can be specified by percentage of votes or absolute \n",
    "                              number of votes\n",
    "    :param label_column:      ground-truth (y) labels\n",
    "    :param stat_filters:      statistical functions to apply to the features\n",
    "                              (from sklearn.feature_selection)\n",
    "    :param model_filters:     models to use for feature evaluation, can be specified by\n",
    "                              model name (ex. LinearSVC), formalized json (contains 'CLASS', \n",
    "                              'FIT', 'META') or a path to such json file.\n",
    "    :param max_scaled_scores: produce feature scores table scaled with max_scaler                            \n",
    "    \"\"\"\n",
    "    \n",
    "    # Read input DF\n",
    "    df_path = str(df_artifact)\n",
    "    context.logger.info(f'input dataset {df_path}')\n",
    "    if df_path.endswith('csv'):\n",
    "        df = pd.read_csv(df_path)\n",
    "    elif df_path.endswith('parquet') or df_path.endswith('pq'):\n",
    "        df = pd.read_parquet(df_path)\n",
    "    \n",
    "    # Set feature vector and labels\n",
    "    y = df.pop(label_column)\n",
    "    X = df\n",
    "    \n",
    "    # Create selected statistical estimators\n",
    "    stat_functions_list = {stat_name:SelectKBest(create_class(f'sklearn.feature_selection.{stat_name}'), k) \n",
    "                               for stat_name in stat_filters}\n",
    "    requires_abs = ['chi2']\n",
    "    \n",
    "    # Run statistic filters\n",
    "    selected_features_agg = {}\n",
    "    stats_df = pd.DataFrame(index=X.columns)\n",
    "    for stat_name, stat_func in stat_functions_list.items():\n",
    "        # Compute statistics\n",
    "        params = (X, y) if stat_name in requires_abs else (abs(X), y)\n",
    "        stat = stat_func.fit(*params)\n",
    "        \n",
    "        # Collect stat function results\n",
    "        stat_df = pd.DataFrame(index=X.columns,\n",
    "                               columns=[stat_name],\n",
    "                               data=stat.scores_)\n",
    "        plot_stat(context, stat_name, stat_df)\n",
    "        stats_df = stats_df.join(stat_df)\n",
    "        \n",
    "        # Select K Best features\n",
    "        selected_features = X.columns[stat_func.get_support()]\n",
    "        selected_features_agg[stat_name] = selected_features\n",
    "                \n",
    "    # Create models from class name / json file / json params\n",
    "    all_sklearn_estimators = dict(all_estimators()) if len(model_filters) > 0 else {}\n",
    "    selected_models = {}\n",
    "    for model_name, model in model_filters.items():\n",
    "        if '.json' in model:\n",
    "            current_model = json.load(open(model, 'r'))\n",
    "            ClassifierClass = create_class(current_model[\"META\"][\"class\"])\n",
    "            selected_models[model_name] = ClassifierClass(**current_model[\"CLASS\"])\n",
    "        elif model in all_sklearn_estimators:\n",
    "            selected_models[model_name] = all_sklearn_estimators[model_name]()\n",
    "        else:\n",
    "            try:\n",
    "                current_model = json.loads(model) if isinstance(model, str) else current_model\n",
    "                ClassifierClass = create_class(current_model[\"META\"][\"class\"])\n",
    "                selected_models[model_name] = ClassifierClass(**current_model[\"CLASS\"])\n",
    "            except:\n",
    "                context.logger.info(f'unable to load {model}')\n",
    "        \n",
    "    # Run model filters\n",
    "    models_df = pd.DataFrame(index=X.columns)\n",
    "    for model_name, model in selected_models.items():\n",
    "        # Train model and get feature importance\n",
    "        select_from_model = SelectFromModel(model).fit(X,y)\n",
    "        feature_idx = select_from_model.get_support()\n",
    "        feature_names = X.columns[feature_idx]\n",
    "        selected_features_agg[model_name] = feature_names.tolist()\n",
    "                \n",
    "        # Collect model feature importance\n",
    "        if hasattr(select_from_model.estimator_, 'coef_'):\n",
    "            stat_df = select_from_model.estimator_.coef_\n",
    "        elif hasattr(select_from_model.estimator_, 'feature_importances_'):\n",
    "            stat_df = select_from_model.estimator_.feature_importances_\n",
    "        stat_df = pd.DataFrame(index=X.columns,\n",
    "                               columns=[model_name],\n",
    "                               data=stat_df[0])\n",
    "        models_df = models_df.join(stat_df)\n",
    "\n",
    "        plot_stat(context, model_name, stat_df)\n",
    "    \n",
    "    # Create feature_scores DF with stat & model filters scores\n",
    "    result_matrix_df = pd.concat([stats_df, models_df], axis=1, sort=False)\n",
    "    context.log_dataset(key='feature_scores', \n",
    "                        df=result_matrix_df,\n",
    "                        local_path='feature_scores.parquet',\n",
    "                        format='parquet')\n",
    "    if max_scaled_scores:\n",
    "        normalized_df = result_matrix_df.replace([np.inf, -np.inf], np.nan).values\n",
    "        min_max_scaler = MinMaxScaler()\n",
    "        normalized_df = min_max_scaler.fit_transform(normalized_df)\n",
    "        normalized_df = pd.DataFrame(data=normalized_df,\n",
    "                                     columns=result_matrix_df.columns,\n",
    "                                     index=result_matrix_df.index)\n",
    "        context.log_dataset(key='max_scaled_scores_feature_scores', \n",
    "                            df=normalized_df,\n",
    "                            local_path='max_scaled_scores_feature_scores.parquet',\n",
    "                            format='parquet')\n",
    "    \n",
    "    # Create feature count DataFrame\n",
    "    for test_name in selected_features_agg:\n",
    "        result_matrix_df[test_name] = [1 if x in selected_features_agg[test_name] else 0 for x in X.columns]\n",
    "    result_matrix_df.loc[:,'num_votes'] = result_matrix_df.sum(axis=1)\n",
    "    context.log_dataset(key='selected_features_count', \n",
    "                        df=result_matrix_df,\n",
    "                        local_path='selected_features_count.parquet',\n",
    "                        format='parquet')\n",
    "    \n",
    "    # How many votes are needed for a feature to be selected?\n",
    "    if isinstance(min_votes, int):\n",
    "        votes_needed = min_votes\n",
    "    else:\n",
    "        num_filters = len(stat_filters) + len(model_filters)\n",
    "        votes_needed = int(np.floor(num_filters * max(min(min_votes, 1), 0)))\n",
    "    context.logger.info(f'votes needed to be selected: {votes_needed}')\n",
    "    \n",
    "    # Create final feature dataframe\n",
    "    selected_features = result_matrix_df[result_matrix_df.num_votes>=votes_needed].index.tolist()\n",
    "    good_feature_df = df.loc[:, selected_features]\n",
    "    final_df = pd.concat([good_feature_df,y], axis=1)\n",
    "    context.log_dataset(key='selected_features',\n",
    "                        df=final_df,\n",
    "                        local_path='selected_features.parquet',\n",
    "                        format='parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nuclio: end-code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlrun import code_to_function, mount_v3io, mlconf, NewTask, run_local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlconf.artifact_path = os.path.abspath('./artifacts')\n",
    "mlconf.db_path = 'http://mlrun-api:8080'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = NewTask(params={'k': 2,\n",
    "                       'min_votes': 0.3,\n",
    "                       'label_column': 'is_error'},\n",
    "               inputs={'df_artifact': '/User/demo-network-operations/data/metrics.parquet'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-04-12 12:28:08,160 starting run feature_selection uid=558aa6cf639d4e9eab6c8d6020f45962  -> http://10.194.95.255:8080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/conda/lib/python3.6/site-packages/sklearn/feature_selection/_univariate_selection.py:115: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  f = msb / msw\n",
      "posx and posy should be finite values\n",
      "/conda/lib/python3.6/site-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/conda/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-04-12 12:28:08,333 input dataset /User/demo-network-operations/data/metrics.parquet\n",
      "[mlrun] 2020-04-12 12:28:10,756 log artifact f_classif at /User/functions/feature_selection/artifacts/plots/feature_selection/f_classif.html, size: 22020, db: Y\n",
      "[mlrun] 2020-04-12 12:28:13,489 log artifact mutual_info_classif at /User/functions/feature_selection/artifacts/plots/feature_selection/mutual_info_classif.html, size: 13738, db: Y\n",
      "[mlrun] 2020-04-12 12:28:14,731 log artifact chi2 at /User/functions/feature_selection/artifacts/plots/feature_selection/chi2.html, size: 21563, db: Y\n",
      "[mlrun] 2020-04-12 12:28:16,084 log artifact f_regression at /User/functions/feature_selection/artifacts/plots/feature_selection/f_regression.html, size: 23635, db: Y\n",
      "[mlrun] 2020-04-12 12:28:17,914 log artifact LinearSVC at /User/functions/feature_selection/artifacts/plots/feature_selection/LinearSVC.html, size: 13296, db: Y\n",
      "[mlrun] 2020-04-12 12:28:22,923 log artifact LogisticRegression at /User/functions/feature_selection/artifacts/plots/feature_selection/LogisticRegression.html, size: 17697, db: Y\n",
      "[mlrun] 2020-04-12 12:28:24,850 log artifact ExtraTreesClassifier at /User/functions/feature_selection/artifacts/plots/feature_selection/ExtraTreesClassifier.html, size: 14059, db: Y\n",
      "[mlrun] 2020-04-12 12:28:25,395 log artifact feature_scores at /User/functions/feature_selection/artifacts/feature_scores.parquet, size: 6739, db: Y\n",
      "[mlrun] 2020-04-12 12:28:25,816 log artifact max_scaled_scores_feature_scores at /User/functions/feature_selection/artifacts/max_scaled_scores_feature_scores.parquet, size: 6682, db: Y\n",
      "[mlrun] 2020-04-12 12:28:26,145 log artifact selected_features_count at /User/functions/feature_selection/artifacts/selected_features_count.parquet, size: 7068, db: Y\n",
      "[mlrun] 2020-04-12 12:28:26,149 votes needed to be selected: 2\n",
      "[mlrun] 2020-04-12 12:28:26,480 log artifact selected_features at /User/functions/feature_selection/artifacts/selected_features.parquet, size: 92130, db: Y\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"558aa6cf639d4e9eab6c8d6020f45962\"><a href=\"https://mlrun-ui.default-tenant.app.dev6.lab.iguazeng.com/projects/default/jobs/558aa6cf639d4e9eab6c8d6020f45962/info\" target=\"_blank\" >...20f45962</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Apr 12 12:28:08</td>\n",
       "      <td>completed</td>\n",
       "      <td>feature_selection</td>\n",
       "      <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=handler</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=jupyter-76c7c6dbb5-vsqfd</div></td>\n",
       "      <td><div title=\"/User/demo-network-operations/data/metrics.parquet\">df_artifact</div></td>\n",
       "      <td><div class=\"dictlist\">k=2</div><div class=\"dictlist\">min_votes=0.3</div><div class=\"dictlist\">label_column=is_error</div></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/f_classif.html\">f_classif</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/mutual_info_classif.html\">mutual_info_classif</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/chi2.html\">chi2</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/f_regression.html\">f_regression</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/LinearSVC.html\">LinearSVC</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/LogisticRegression.html\">LogisticRegression</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result35720438\" title=\"/files/functions/feature_selection/artifacts/plots/feature_selection/ExtraTreesClassifier.html\">ExtraTreesClassifier</div><div title=\"/User/functions/feature_selection/artifacts/feature_scores.parquet\">feature_scores</div><div title=\"/User/functions/feature_selection/artifacts/max_scaled_scores_feature_scores.parquet\">max_scaled_scores_feature_scores</div><div title=\"/User/functions/feature_selection/artifacts/selected_features_count.parquet\">selected_features_count</div><div title=\"/User/functions/feature_selection/artifacts/selected_features.parquet\">selected_features</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result35720438-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result35720438-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result35720438\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result35720438-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run 558aa6cf639d4e9eab6c8d6020f45962 --project default , !mlrun logs 558aa6cf639d4e9eab6c8d6020f45962 --project default\n",
      "[mlrun] 2020-04-12 12:28:26,989 run executed, status=completed\n"
     ]
    }
   ],
   "source": [
    "runl = run_local(task=task,\n",
    "          name='feature_selection',\n",
    "          handler=feature_selection,\n",
    "          artifact_path=os.path.join(os.path.abspath('./'), 'artifacts'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Job Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-05-01 22:10:35,067 function spec saved to path: function.yaml\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<mlrun.runtimes.kubejob.KubejobRuntime at 0x7f7380b0ecc0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn = code_to_function(name='feature_selection',\n",
    "                      handler='feature_selection')\n",
    "fn.spec.default_handler = 'feature_selection'\n",
    "fn.spec.description = \"Select features through multiple Statistical and Model filters\"\n",
    "fn.metadata.categories = ['data-prep', 'ml']\n",
    "fn.metadata.labels = {\"author\": \"orz\"}\n",
    "fn.export('function.yaml')\n",
    "fn.apply(mount_v3io())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mlrun] 2020-04-12 13:24:01,198 starting run feature-selection-feature_selection uid=27215a4474d34055ba376f15fe092a58  -> http://10.194.95.255:8080\n",
      "[mlrun] 2020-04-12 13:24:02,000 Job is running in the background, pod: feature-selection-feature-selection-8rkr5\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/feature_selection/_univariate_selection.py:115: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  f = msb / msw\n",
      "posx and posy should be finite values\n",
      "[mlrun] 2020-04-12 13:24:47,709 input dataset /User/demo-network-operations/data/metrics.parquet\n",
      "[mlrun] 2020-04-12 13:24:49,249 log artifact f_classif at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/f_classif.html, size: 42884, db: Y\n",
      "[mlrun] 2020-04-12 13:24:51,930 log artifact mutual_info_classif at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/mutual_info_classif.html, size: 30250, db: Y\n",
      "[mlrun] 2020-04-12 13:24:53,046 log artifact chi2 at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/chi2.html, size: 42515, db: Y\n",
      "[mlrun] 2020-04-12 13:24:54,161 log artifact f_regression at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/f_regression.html, size: 46123, db: Y\n",
      "[mlrun] 2020-04-12 13:24:57,875 log artifact LinearSVC at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/LinearSVC.html, size: 29556, db: Y\n",
      "[mlrun] 2020-04-12 13:24:59,459 log artifact LogisticRegression at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/LogisticRegression.html, size: 35545, db: Y\n",
      "[mlrun] 2020-04-12 13:25:01,276 log artifact ExtraTreesClassifier at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/ExtraTreesClassifier.html, size: 31583, db: Y\n",
      "[mlrun] 2020-04-12 13:25:01,739 log artifact feature_scores at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/feature_scores.parquet, size: 6736, db: Y\n",
      "[mlrun] 2020-04-12 13:25:01,997 log artifact max_scaled_scores_feature_scores at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/max_scaled_scores_feature_scores.parquet, size: 6674, db: Y\n",
      "[mlrun] 2020-04-12 13:25:02,242 log artifact selected_features_count at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/selected_features_count.parquet, size: 7060, db: Y\n",
      "[mlrun] 2020-04-12 13:25:02,246 votes needed to be selected: 2\n",
      "[mlrun] 2020-04-12 13:25:02,498 log artifact selected_features at /User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/selected_features.parquet, size: 63479, db: Y\n",
      "\n",
      "[mlrun] 2020-04-12 13:25:02,799 run executed, status=completed\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/opt/conda/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "final state: succeeded\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style> \n",
       ".dictlist {\n",
       "  background-color: #b3edff; \n",
       "  text-align: center; \n",
       "  margin: 4px; \n",
       "  border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;}\n",
       ".artifact {\n",
       "  cursor: pointer; \n",
       "  background-color: #ffe6cc; \n",
       "  text-align: left; \n",
       "  margin: 4px; border-radius: 3px; padding: 0px 3px 1px 3px; display: inline-block;\n",
       "}\n",
       "div.block.hidden {\n",
       "  display: none;\n",
       "}\n",
       ".clickable {\n",
       "  cursor: pointer;\n",
       "}\n",
       ".ellipsis {\n",
       "  display: inline-block;\n",
       "  max-width: 60px;\n",
       "  white-space: nowrap;\n",
       "  overflow: hidden;\n",
       "  text-overflow: ellipsis;\n",
       "}\n",
       ".master-wrapper {\n",
       "  display: flex;\n",
       "  flex-flow: row nowrap;\n",
       "  justify-content: flex-start;\n",
       "  align-items: stretch;\n",
       "}\n",
       ".master-tbl {\n",
       "  flex: 3\n",
       "}\n",
       ".master-wrapper > div {\n",
       "  margin: 4px;\n",
       "  padding: 10px;\n",
       "}\n",
       "iframe.fileview {\n",
       "  border: 0 none;\n",
       "  height: 100%;\n",
       "  width: 100%;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       ".pane-header-title {\n",
       "  width: 80%;\n",
       "  font-weight: 500;\n",
       "}\n",
       ".pane-header {\n",
       "  line-height: 1;\n",
       "  background-color: #ffe6cc;\n",
       "  padding: 3px;\n",
       "}\n",
       ".pane-header .close {\n",
       "  font-size: 20px;\n",
       "  font-weight: 700;\n",
       "  float: right;\n",
       "  margin-top: -5px;\n",
       "}\n",
       ".master-wrapper .right-pane {\n",
       "  border: 1px inset silver;\n",
       "  width: 40%;\n",
       "  min-height: 300px;\n",
       "  flex: 3\n",
       "  min-width: 500px;\n",
       "}\n",
       ".master-wrapper * {\n",
       "  box-sizing: border-box;\n",
       "}\n",
       "</style><script>\n",
       "function copyToClipboard(fld) {\n",
       "    if (document.queryCommandSupported && document.queryCommandSupported('copy')) {\n",
       "        var textarea = document.createElement('textarea');\n",
       "        textarea.textContent = fld.innerHTML;\n",
       "        textarea.style.position = 'fixed';\n",
       "        document.body.appendChild(textarea);\n",
       "        textarea.select();\n",
       "\n",
       "        try {\n",
       "            return document.execCommand('copy'); // Security exception may be thrown by some browsers.\n",
       "        } catch (ex) {\n",
       "\n",
       "        } finally {\n",
       "            document.body.removeChild(textarea);\n",
       "        }\n",
       "    }\n",
       "}\n",
       "function expandPanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName');\n",
       "  console.log(el.title);\n",
       "\n",
       "  document.querySelector(panelName + \"-title\").innerHTML = el.title\n",
       "  iframe = document.querySelector(panelName + \"-body\");\n",
       "  \n",
       "  const tblcss = `<style> body { font-family: Arial, Helvetica, sans-serif;}\n",
       "    #csv { margin-bottom: 15px; }\n",
       "    #csv table { border-collapse: collapse;}\n",
       "    #csv table td { padding: 4px 8px; border: 1px solid silver;} </style>`;\n",
       "\n",
       "  function csvToHtmlTable(str) {\n",
       "    return '<div id=\"csv\"><table><tr><td>' +  str.replace(/[\\n\\r]+$/g, '').replace(/[\\n\\r]+/g, '</td></tr><tr><td>')\n",
       "      .replace(/,/g, '</td><td>') + '</td></tr></table></div>';\n",
       "  }\n",
       "  \n",
       "  function reqListener () {\n",
       "    if (el.title.endsWith(\".csv\")) {\n",
       "      iframe.setAttribute(\"srcdoc\", tblcss + csvToHtmlTable(this.responseText));\n",
       "    } else {\n",
       "      iframe.setAttribute(\"srcdoc\", this.responseText);\n",
       "    }  \n",
       "    console.log(this.responseText);\n",
       "  }\n",
       "\n",
       "  const oReq = new XMLHttpRequest();\n",
       "  oReq.addEventListener(\"load\", reqListener);\n",
       "  oReq.open(\"GET\", el.title);\n",
       "  oReq.send();\n",
       "  \n",
       "  \n",
       "  //iframe.src = el.title;\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.remove(\"hidden\");\n",
       "  }\n",
       "}\n",
       "function closePanel(el) {\n",
       "  const panelName = \"#\" + el.getAttribute('paneName')\n",
       "  const resultPane = document.querySelector(panelName + \"-pane\");\n",
       "  if (!resultPane.classList.contains(\"hidden\")) {\n",
       "    resultPane.classList.add(\"hidden\");\n",
       "  }\n",
       "}\n",
       "\n",
       "</script>\n",
       "<div class=\"master-wrapper\">\n",
       "  <div class=\"block master-tbl\"><div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>project</th>\n",
       "      <th>uid</th>\n",
       "      <th>iter</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>labels</th>\n",
       "      <th>inputs</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "      <th>artifacts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>default</td>\n",
       "      <td><div title=\"27215a4474d34055ba376f15fe092a58\"><a href=\"https://mlrun-ui.default-tenant.app.dev6.lab.iguazeng.com/projects/default/jobs/27215a4474d34055ba376f15fe092a58/info\" target=\"_blank\" >...fe092a58</a></div></td>\n",
       "      <td>0</td>\n",
       "      <td>Apr 12 13:24:47</td>\n",
       "      <td>completed</td>\n",
       "      <td>feature-selection-feature_selection</td>\n",
       "      <td><div class=\"dictlist\">host=feature-selection-feature-selection-8rkr5</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">v3io_user=admin</div></td>\n",
       "      <td><div title=\"/User/demo-network-operations/data/metrics.parquet\">df_artifact</div></td>\n",
       "      <td><div class=\"dictlist\">k=2</div><div class=\"dictlist\">label_column=is_error</div><div class=\"dictlist\">min_votes=0.3</div></td>\n",
       "      <td></td>\n",
       "      <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/f_classif.html\">f_classif</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/mutual_info_classif.html\">mutual_info_classif</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/chi2.html\">chi2</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/f_regression.html\">f_regression</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/LinearSVC.html\">LinearSVC</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/LogisticRegression.html\">LogisticRegression</div><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result4bf3ef43\" title=\"/files/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/plots/feature_selection/ExtraTreesClassifier.html\">ExtraTreesClassifier</div><div title=\"/User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/feature_scores.parquet\">feature_scores</div><div title=\"/User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/max_scaled_scores_feature_scores.parquet\">max_scaled_scores_feature_scores</div><div title=\"/User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/selected_features_count.parquet\">selected_features_count</div><div title=\"/User/functions/feature_selection/artifacts/27215a4474d34055ba376f15fe092a58/selected_features.parquet\">selected_features</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div></div>\n",
       "  <div id=\"result4bf3ef43-pane\" class=\"right-pane block hidden\">\n",
       "    <div class=\"pane-header\">\n",
       "      <span id=\"result4bf3ef43-title\" class=\"pane-header-title\">Title</span>\n",
       "      <span onclick=\"closePanel(this)\" paneName=\"result4bf3ef43\" class=\"close clickable\">&times;</span>\n",
       "    </div>\n",
       "    <iframe class=\"fileview\" id=\"result4bf3ef43-body\"></iframe>\n",
       "  </div>\n",
       "</div>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to track results use .show() or .logs() or in CLI: \n",
      "!mlrun get run 27215a4474d34055ba376f15fe092a58  , !mlrun logs 27215a4474d34055ba376f15fe092a58 \n",
      "[mlrun] 2020-04-12 13:25:13,202 run executed, status=completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<mlrun.model.RunObject at 0x7f4ebc1210b8>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fn.run(task)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>cpu_utilization</th>\n",
       "      <th>cpu_utilization_is_error</th>\n",
       "      <th>latency</th>\n",
       "      <th>latency_is_error</th>\n",
       "      <th>packet_loss</th>\n",
       "      <th>packet_loss_is_error</th>\n",
       "      <th>throughput</th>\n",
       "      <th>throughput_is_error</th>\n",
       "      <th>is_error</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th>company</th>\n",
       "      <th>data_center</th>\n",
       "      <th>device</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-03-29 19:22:10.106</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">Johnson-Morgan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Glenn_Port</th>\n",
       "      <th>6625659405376</th>\n",
       "      <td>69.242828</td>\n",
       "      <td>False</td>\n",
       "      <td>10.122039</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>259.837367</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0306839395881</th>\n",
       "      <td>55.539413</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>2.338992</td>\n",
       "      <td>False</td>\n",
       "      <td>277.264491</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Baker_Locks</th>\n",
       "      <th>9686333640344</th>\n",
       "      <td>54.234624</td>\n",
       "      <td>False</td>\n",
       "      <td>1.099465</td>\n",
       "      <td>False</td>\n",
       "      <td>0.535960</td>\n",
       "      <td>False</td>\n",
       "      <td>214.538381</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6135824620701</th>\n",
       "      <td>83.379805</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>0.973531</td>\n",
       "      <td>False</td>\n",
       "      <td>210.010873</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Romero-Perry</th>\n",
       "      <th>Kim_Locks</th>\n",
       "      <th>9598503476170</th>\n",
       "      <td>81.713657</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>0.817911</td>\n",
       "      <td>False</td>\n",
       "      <td>239.972481</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-03-29 20:22:10.106</th>\n",
       "      <th>Johnson-Morgan</th>\n",
       "      <th>Baker_Locks</th>\n",
       "      <th>6135824620701</th>\n",
       "      <td>66.323627</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>0.169410</td>\n",
       "      <td>False</td>\n",
       "      <td>208.415531</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Romero-Perry</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Kim_Locks</th>\n",
       "      <th>9598503476170</th>\n",
       "      <td>54.032799</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>249.860566</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6733246376493</th>\n",
       "      <td>76.085686</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>233.559866</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Moore_Gateway</th>\n",
       "      <th>5925984923860</th>\n",
       "      <td>67.469579</td>\n",
       "      <td>False</td>\n",
       "      <td>0.156262</td>\n",
       "      <td>False</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>253.169495</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4969195108444</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>270.954699</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5768 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    cpu_utilization  \\\n",
       "timestamp               company        data_center   device                           \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376        69.242828   \n",
       "                                                     0306839395881        55.539413   \n",
       "                                       Baker_Locks   9686333640344        54.234624   \n",
       "                                                     6135824620701        83.379805   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170        81.713657   \n",
       "...                                                                             ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701        66.323627   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170        54.032799   \n",
       "                                                     6733246376493        76.085686   \n",
       "                                       Moore_Gateway 5925984923860        67.469579   \n",
       "                                                     4969195108444       100.000000   \n",
       "\n",
       "                                                                    cpu_utilization_is_error  \\\n",
       "timestamp               company        data_center   device                                    \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376                     False   \n",
       "                                                     0306839395881                     False   \n",
       "                                       Baker_Locks   9686333640344                     False   \n",
       "                                                     6135824620701                     False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                     False   \n",
       "...                                                                                      ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701                     False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                     False   \n",
       "                                                     6733246376493                     False   \n",
       "                                       Moore_Gateway 5925984923860                     False   \n",
       "                                                     4969195108444                      True   \n",
       "\n",
       "                                                                       latency  \\\n",
       "timestamp               company        data_center   device                      \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376   10.122039   \n",
       "                                                     0306839395881    0.000000   \n",
       "                                       Baker_Locks   9686333640344    1.099465   \n",
       "                                                     6135824620701    0.000000   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170    0.000000   \n",
       "...                                                                        ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701    0.000000   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170    0.000000   \n",
       "                                                     6733246376493    0.000000   \n",
       "                                       Moore_Gateway 5925984923860    0.156262   \n",
       "                                                     4969195108444  100.000000   \n",
       "\n",
       "                                                                    latency_is_error  \\\n",
       "timestamp               company        data_center   device                            \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376             False   \n",
       "                                                     0306839395881             False   \n",
       "                                       Baker_Locks   9686333640344             False   \n",
       "                                                     6135824620701             False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170             False   \n",
       "...                                                                              ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701             False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170             False   \n",
       "                                                     6733246376493             False   \n",
       "                                       Moore_Gateway 5925984923860             False   \n",
       "                                                     4969195108444              True   \n",
       "\n",
       "                                                                    packet_loss  \\\n",
       "timestamp               company        data_center   device                       \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376     0.000000   \n",
       "                                                     0306839395881     2.338992   \n",
       "                                       Baker_Locks   9686333640344     0.535960   \n",
       "                                                     6135824620701     0.973531   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     0.817911   \n",
       "...                                                                         ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701     0.169410   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     0.000000   \n",
       "                                                     6733246376493     0.000000   \n",
       "                                       Moore_Gateway 5925984923860     0.000000   \n",
       "                                                     4969195108444    50.000000   \n",
       "\n",
       "                                                                    packet_loss_is_error  \\\n",
       "timestamp               company        data_center   device                                \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376                 False   \n",
       "                                                     0306839395881                 False   \n",
       "                                       Baker_Locks   9686333640344                 False   \n",
       "                                                     6135824620701                 False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                 False   \n",
       "...                                                                                  ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701                 False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                 False   \n",
       "                                                     6733246376493                 False   \n",
       "                                       Moore_Gateway 5925984923860                 False   \n",
       "                                                     4969195108444                  True   \n",
       "\n",
       "                                                                    throughput  \\\n",
       "timestamp               company        data_center   device                      \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376  259.837367   \n",
       "                                                     0306839395881  277.264491   \n",
       "                                       Baker_Locks   9686333640344  214.538381   \n",
       "                                                     6135824620701  210.010873   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170  239.972481   \n",
       "...                                                                        ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701  208.415531   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170  249.860566   \n",
       "                                                     6733246376493  233.559866   \n",
       "                                       Moore_Gateway 5925984923860  253.169495   \n",
       "                                                     4969195108444  270.954699   \n",
       "\n",
       "                                                                    throughput_is_error  \\\n",
       "timestamp               company        data_center   device                               \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376                False   \n",
       "                                                     0306839395881                False   \n",
       "                                       Baker_Locks   9686333640344                False   \n",
       "                                                     6135824620701                False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                False   \n",
       "...                                                                                 ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701                False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                False   \n",
       "                                                     6733246376493                False   \n",
       "                                       Moore_Gateway 5925984923860                False   \n",
       "                                                     4969195108444                False   \n",
       "\n",
       "                                                                    is_error  \n",
       "timestamp               company        data_center   device                   \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376     False  \n",
       "                                                     0306839395881     False  \n",
       "                                       Baker_Locks   9686333640344     False  \n",
       "                                                     6135824620701     False  \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     False  \n",
       "...                                                                      ...  \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701     False  \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     False  \n",
       "                                                     6733246376493     False  \n",
       "                                       Moore_Gateway 5925984923860     False  \n",
       "                                                     4969195108444     False  \n",
       "\n",
       "[5768 rows x 9 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_parquet(runl.spec.inputs['df_artifact'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f_classif</th>\n",
       "      <th>mutual_info_classif</th>\n",
       "      <th>chi2</th>\n",
       "      <th>f_regression</th>\n",
       "      <th>LinearSVC</th>\n",
       "      <th>LogisticRegression</th>\n",
       "      <th>ExtraTreesClassifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cpu_utilization</th>\n",
       "      <td>2.582004e+03</td>\n",
       "      <td>0.189574</td>\n",
       "      <td>4639.351240</td>\n",
       "      <td>2.582004e+03</td>\n",
       "      <td>-0.139358</td>\n",
       "      <td>-0.154484</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cpu_utilization_is_error</th>\n",
       "      <td>1.025946e+04</td>\n",
       "      <td>0.203149</td>\n",
       "      <td>3243.240561</td>\n",
       "      <td>1.025946e+04</td>\n",
       "      <td>0.025081</td>\n",
       "      <td>0.056447</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency</th>\n",
       "      <td>9.161905e+03</td>\n",
       "      <td>0.200192</td>\n",
       "      <td>263740.808988</td>\n",
       "      <td>9.161905e+03</td>\n",
       "      <td>0.068929</td>\n",
       "      <td>0.103105</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_is_error</th>\n",
       "      <td>1.579942e+04</td>\n",
       "      <td>0.221002</td>\n",
       "      <td>3769.368661</td>\n",
       "      <td>1.579942e+04</td>\n",
       "      <td>0.025081</td>\n",
       "      <td>0.049925</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>packet_loss</th>\n",
       "      <td>1.426352e+04</td>\n",
       "      <td>0.222967</td>\n",
       "      <td>157171.402720</td>\n",
       "      <td>1.426352e+04</td>\n",
       "      <td>0.138662</td>\n",
       "      <td>0.201053</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>packet_loss_is_error</th>\n",
       "      <td>3.064512e+04</td>\n",
       "      <td>0.243446</td>\n",
       "      <td>4391.686839</td>\n",
       "      <td>3.064512e+04</td>\n",
       "      <td>0.083481</td>\n",
       "      <td>0.102485</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>throughput</th>\n",
       "      <td>1.937176e+04</td>\n",
       "      <td>0.238092</td>\n",
       "      <td>115449.931690</td>\n",
       "      <td>1.937176e+04</td>\n",
       "      <td>-0.028992</td>\n",
       "      <td>-0.163124</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>throughput_is_error</th>\n",
       "      <td>inf</td>\n",
       "      <td>0.282466</td>\n",
       "      <td>5298.000000</td>\n",
       "      <td>-6.491939e+18</td>\n",
       "      <td>0.996251</td>\n",
       "      <td>5.250064</td>\n",
       "      <td>0.006194</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             f_classif  mutual_info_classif           chi2  \\\n",
       "cpu_utilization           2.582004e+03             0.189574    4639.351240   \n",
       "cpu_utilization_is_error  1.025946e+04             0.203149    3243.240561   \n",
       "latency                   9.161905e+03             0.200192  263740.808988   \n",
       "latency_is_error          1.579942e+04             0.221002    3769.368661   \n",
       "packet_loss               1.426352e+04             0.222967  157171.402720   \n",
       "packet_loss_is_error      3.064512e+04             0.243446    4391.686839   \n",
       "throughput                1.937176e+04             0.238092  115449.931690   \n",
       "throughput_is_error                inf             0.282466    5298.000000   \n",
       "\n",
       "                          f_regression  LinearSVC  LogisticRegression  \\\n",
       "cpu_utilization           2.582004e+03  -0.139358           -0.154484   \n",
       "cpu_utilization_is_error  1.025946e+04   0.025081            0.056447   \n",
       "latency                   9.161905e+03   0.068929            0.103105   \n",
       "latency_is_error          1.579942e+04   0.025081            0.049925   \n",
       "packet_loss               1.426352e+04   0.138662            0.201053   \n",
       "packet_loss_is_error      3.064512e+04   0.083481            0.102485   \n",
       "throughput                1.937176e+04  -0.028992           -0.163124   \n",
       "throughput_is_error      -6.491939e+18   0.996251            5.250064   \n",
       "\n",
       "                          ExtraTreesClassifier  \n",
       "cpu_utilization                       0.006194  \n",
       "cpu_utilization_is_error              0.006194  \n",
       "latency                               0.006194  \n",
       "latency_is_error                      0.006194  \n",
       "packet_loss                           0.006194  \n",
       "packet_loss_is_error                  0.006194  \n",
       "throughput                            0.006194  \n",
       "throughput_is_error                   0.006194  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_parquet(runl.outputs['feature_scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f_classif</th>\n",
       "      <th>mutual_info_classif</th>\n",
       "      <th>chi2</th>\n",
       "      <th>f_regression</th>\n",
       "      <th>LinearSVC</th>\n",
       "      <th>LogisticRegression</th>\n",
       "      <th>ExtraTreesClassifier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cpu_utilization</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005359</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cpu_utilization_is_error</th>\n",
       "      <td>0.273578</td>\n",
       "      <td>0.146140</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.144802</td>\n",
       "      <td>0.040562</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency</th>\n",
       "      <td>0.234468</td>\n",
       "      <td>0.114307</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.183414</td>\n",
       "      <td>0.049182</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_is_error</th>\n",
       "      <td>0.470989</td>\n",
       "      <td>0.338327</td>\n",
       "      <td>0.002020</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.144802</td>\n",
       "      <td>0.039357</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>packet_loss</th>\n",
       "      <td>0.416259</td>\n",
       "      <td>0.359483</td>\n",
       "      <td>0.590901</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.244820</td>\n",
       "      <td>0.067276</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>packet_loss_is_error</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.579949</td>\n",
       "      <td>0.004409</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.196229</td>\n",
       "      <td>0.049067</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>throughput</th>\n",
       "      <td>0.598285</td>\n",
       "      <td>0.522302</td>\n",
       "      <td>0.430740</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.097186</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>throughput_is_error</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.007888</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          f_classif  mutual_info_classif      chi2  \\\n",
       "cpu_utilization            0.000000             0.000000  0.005359   \n",
       "cpu_utilization_is_error   0.273578             0.146140  0.000000   \n",
       "latency                    0.234468             0.114307  1.000000   \n",
       "latency_is_error           0.470989             0.338327  0.002020   \n",
       "packet_loss                0.416259             0.359483  0.590901   \n",
       "packet_loss_is_error       1.000000             0.579949  0.004409   \n",
       "throughput                 0.598285             0.522302  0.430740   \n",
       "throughput_is_error             NaN             1.000000  0.007888   \n",
       "\n",
       "                          f_regression  LinearSVC  LogisticRegression  \\\n",
       "cpu_utilization                    1.0   0.000000            0.001596   \n",
       "cpu_utilization_is_error           1.0   0.144802            0.040562   \n",
       "latency                            1.0   0.183414            0.049182   \n",
       "latency_is_error                   1.0   0.144802            0.039357   \n",
       "packet_loss                        1.0   0.244820            0.067276   \n",
       "packet_loss_is_error               1.0   0.196229            0.049067   \n",
       "throughput                         1.0   0.097186            0.000000   \n",
       "throughput_is_error                0.0   1.000000            1.000000   \n",
       "\n",
       "                          ExtraTreesClassifier  \n",
       "cpu_utilization                            0.0  \n",
       "cpu_utilization_is_error                   0.0  \n",
       "latency                                    0.0  \n",
       "latency_is_error                           0.0  \n",
       "packet_loss                                0.0  \n",
       "packet_loss_is_error                       0.0  \n",
       "throughput                                 0.0  \n",
       "throughput_is_error                        0.0  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_parquet(runl.outputs['max_scaled_scores_feature_scores'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f_classif</th>\n",
       "      <th>mutual_info_classif</th>\n",
       "      <th>chi2</th>\n",
       "      <th>f_regression</th>\n",
       "      <th>LinearSVC</th>\n",
       "      <th>LogisticRegression</th>\n",
       "      <th>ExtraTreesClassifier</th>\n",
       "      <th>num_votes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cpu_utilization</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cpu_utilization_is_error</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>latency_is_error</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>packet_loss</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>packet_loss_is_error</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>throughput</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>throughput_is_error</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          f_classif  mutual_info_classif  chi2  f_regression  \\\n",
       "cpu_utilization                   0                    0     0             0   \n",
       "cpu_utilization_is_error          0                    0     0             0   \n",
       "latency                           0                    0     1             0   \n",
       "latency_is_error                  0                    0     0             0   \n",
       "packet_loss                       0                    0     1             0   \n",
       "packet_loss_is_error              1                    1     0             1   \n",
       "throughput                        0                    0     0             1   \n",
       "throughput_is_error               1                    1     0             0   \n",
       "\n",
       "                          LinearSVC  LogisticRegression  ExtraTreesClassifier  \\\n",
       "cpu_utilization                   0                   0                     0   \n",
       "cpu_utilization_is_error          0                   0                     0   \n",
       "latency                           0                   0                     0   \n",
       "latency_is_error                  0                   0                     1   \n",
       "packet_loss                       0                   0                     1   \n",
       "packet_loss_is_error              0                   0                     1   \n",
       "throughput                        0                   0                     1   \n",
       "throughput_is_error               1                   1                     1   \n",
       "\n",
       "                          num_votes  \n",
       "cpu_utilization                   0  \n",
       "cpu_utilization_is_error          0  \n",
       "latency                           1  \n",
       "latency_is_error                  1  \n",
       "packet_loss                       2  \n",
       "packet_loss_is_error              4  \n",
       "throughput                        2  \n",
       "throughput_is_error               5  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_parquet(runl.outputs['selected_features_count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>packet_loss</th>\n",
       "      <th>packet_loss_is_error</th>\n",
       "      <th>throughput</th>\n",
       "      <th>throughput_is_error</th>\n",
       "      <th>is_error</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th>company</th>\n",
       "      <th>data_center</th>\n",
       "      <th>device</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-03-29 19:22:10.106</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">Johnson-Morgan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Glenn_Port</th>\n",
       "      <th>6625659405376</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>259.837367</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0306839395881</th>\n",
       "      <td>2.338992</td>\n",
       "      <td>False</td>\n",
       "      <td>277.264491</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Baker_Locks</th>\n",
       "      <th>9686333640344</th>\n",
       "      <td>0.535960</td>\n",
       "      <td>False</td>\n",
       "      <td>214.538381</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6135824620701</th>\n",
       "      <td>0.973531</td>\n",
       "      <td>False</td>\n",
       "      <td>210.010873</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Romero-Perry</th>\n",
       "      <th>Kim_Locks</th>\n",
       "      <th>9598503476170</th>\n",
       "      <td>0.817911</td>\n",
       "      <td>False</td>\n",
       "      <td>239.972481</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2020-03-29 20:22:10.106</th>\n",
       "      <th>Johnson-Morgan</th>\n",
       "      <th>Baker_Locks</th>\n",
       "      <th>6135824620701</th>\n",
       "      <td>0.169410</td>\n",
       "      <td>False</td>\n",
       "      <td>208.415531</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Romero-Perry</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Kim_Locks</th>\n",
       "      <th>9598503476170</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>249.860566</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6733246376493</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>233.559866</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Moore_Gateway</th>\n",
       "      <th>5925984923860</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>253.169495</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4969195108444</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>270.954699</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5768 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    packet_loss  \\\n",
       "timestamp               company        data_center   device                       \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376     0.000000   \n",
       "                                                     0306839395881     2.338992   \n",
       "                                       Baker_Locks   9686333640344     0.535960   \n",
       "                                                     6135824620701     0.973531   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     0.817911   \n",
       "...                                                                         ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701     0.169410   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     0.000000   \n",
       "                                                     6733246376493     0.000000   \n",
       "                                       Moore_Gateway 5925984923860     0.000000   \n",
       "                                                     4969195108444    50.000000   \n",
       "\n",
       "                                                                    packet_loss_is_error  \\\n",
       "timestamp               company        data_center   device                                \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376                 False   \n",
       "                                                     0306839395881                 False   \n",
       "                                       Baker_Locks   9686333640344                 False   \n",
       "                                                     6135824620701                 False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                 False   \n",
       "...                                                                                  ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701                 False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                 False   \n",
       "                                                     6733246376493                 False   \n",
       "                                       Moore_Gateway 5925984923860                 False   \n",
       "                                                     4969195108444                  True   \n",
       "\n",
       "                                                                    throughput  \\\n",
       "timestamp               company        data_center   device                      \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376  259.837367   \n",
       "                                                     0306839395881  277.264491   \n",
       "                                       Baker_Locks   9686333640344  214.538381   \n",
       "                                                     6135824620701  210.010873   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170  239.972481   \n",
       "...                                                                        ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701  208.415531   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170  249.860566   \n",
       "                                                     6733246376493  233.559866   \n",
       "                                       Moore_Gateway 5925984923860  253.169495   \n",
       "                                                     4969195108444  270.954699   \n",
       "\n",
       "                                                                    throughput_is_error  \\\n",
       "timestamp               company        data_center   device                               \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376                False   \n",
       "                                                     0306839395881                False   \n",
       "                                       Baker_Locks   9686333640344                False   \n",
       "                                                     6135824620701                False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                False   \n",
       "...                                                                                 ...   \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701                False   \n",
       "                        Romero-Perry   Kim_Locks     9598503476170                False   \n",
       "                                                     6733246376493                False   \n",
       "                                       Moore_Gateway 5925984923860                False   \n",
       "                                                     4969195108444                False   \n",
       "\n",
       "                                                                    is_error  \n",
       "timestamp               company        data_center   device                   \n",
       "2020-03-29 19:22:10.106 Johnson-Morgan Glenn_Port    6625659405376     False  \n",
       "                                                     0306839395881     False  \n",
       "                                       Baker_Locks   9686333640344     False  \n",
       "                                                     6135824620701     False  \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     False  \n",
       "...                                                                      ...  \n",
       "2020-03-29 20:22:10.106 Johnson-Morgan Baker_Locks   6135824620701     False  \n",
       "                        Romero-Perry   Kim_Locks     9598503476170     False  \n",
       "                                                     6733246376493     False  \n",
       "                                       Moore_Gateway 5925984923860     False  \n",
       "                                                     4969195108444     False  \n",
       "\n",
       "[5768 rows x 5 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_parquet(runl.outputs['selected_features'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}