Check Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check Model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from Survival.Utils import load_val_data\n",
    "from Survival.Utils import calc_scores\n",
    "from Survival.FeatureEngineer import FeatureEngineer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current dataset: pancreatitis\n",
      "---------------------------------------------\n",
      "fold 0\n",
      "1219 11:29:06\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/fancyimpute/soft_impute.py:100: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  return (np.sqrt(ssd) / old_norm) < self.convergence_threshold\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------\n",
      "fold 1\n",
      "1219 11:29:14\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 2\n",
      "1219 11:29:21\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 3\n",
      "1219 11:29:29\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 4\n",
      "1219 11:29:36\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 5\n",
      "1219 11:29:43\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 6\n",
      "1219 11:29:50\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 7\n",
      "1219 11:29:57\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 8\n",
      "1219 11:30:04\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 9\n",
      "1219 11:30:11\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 10\n",
      "1219 11:30:17\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 11\n",
      "1219 11:30:24\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 12\n",
      "1219 11:30:31\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 13\n",
      "1219 11:30:38\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 14\n",
      "1219 11:30:45\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 15\n",
      "1219 11:30:51\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 16\n",
      "1219 11:30:58\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 17\n",
      "1219 11:31:05\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 18\n",
      "1219 11:31:12\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 19\n",
      "1219 11:31:19\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 20\n",
      "1219 11:31:26\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 21\n",
      "1219 11:31:33\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 22\n",
      "1219 11:31:40\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 23\n",
      "1219 11:31:46\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 24\n",
      "1219 11:31:55\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 25\n",
      "1219 11:32:03\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 26\n",
      "1219 11:32:11\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 27\n",
      "1219 11:32:18\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 28\n",
      "1219 11:32:25\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 29\n",
      "1219 11:32:33\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 30\n",
      "1219 11:32:40\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 31\n",
      "1219 11:32:48\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 32\n",
      "1219 11:32:56\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 33\n",
      "1219 11:33:03\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 34\n",
      "1219 11:33:10\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 35\n",
      "1219 11:33:18\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 36\n",
      "1219 11:33:25\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 37\n",
      "1219 11:33:32\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 38\n",
      "1219 11:33:39\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 39\n",
      "1219 11:33:47\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 40\n",
      "1219 11:33:55\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 41\n",
      "1219 11:34:01\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 42\n",
      "1219 11:34:08\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 43\n",
      "1219 11:34:15\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 44\n",
      "1219 11:34:23\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 45\n",
      "1219 11:34:30\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 46\n",
      "1219 11:34:37\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 47\n",
      "1219 11:34:44\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 48\n",
      "1219 11:34:51\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 49\n",
      "1219 11:34:57\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 5 times.\n",
      "[LOG]- Number of events that ever occurred: 1487\n",
      "[LOG]- Number of events that occurred for more than 5 times: 612\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 2 times.\n",
      "[LOG]After one-hot encoding, number of feature: 997\n",
      "[LOG]Among them, number of categorical feature: 641\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "current dataset: ich\n",
      "---------------------------------------------\n",
      "fold 0\n",
      "1219 11:35:04\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LOG]Event filtering. Remove events that occurred for less than 7 times.\n",
      "[LOG]- Number of events that ever occurred: 1896\n",
      "[LOG]- Number of events that occurred for more than 7 times: 817\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 3 times.\n",
      "[LOG]After one-hot encoding, number of feature: 1709\n",
      "[LOG]Among them, number of categorical feature: 1277\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 1\n",
      "1219 11:37:25\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 7 times.\n",
      "[LOG]- Number of events that ever occurred: 1896\n",
      "[LOG]- Number of events that occurred for more than 7 times: 817\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 3 times.\n",
      "[LOG]After one-hot encoding, number of feature: 1709\n",
      "[LOG]Among them, number of categorical feature: 1277\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 2\n",
      "1219 11:39:45\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 7 times.\n",
      "[LOG]- Number of events that ever occurred: 1896\n",
      "[LOG]- Number of events that occurred for more than 7 times: 817\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 3 times.\n",
      "[LOG]After one-hot encoding, number of feature: 1709\n",
      "[LOG]Among them, number of categorical feature: 1277\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 3\n",
      "1219 11:42:02\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 7 times.\n",
      "[LOG]- Number of events that ever occurred: 1896\n",
      "[LOG]- Number of events that occurred for more than 7 times: 817\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 3 times.\n",
      "[LOG]After one-hot encoding, number of feature: 1709\n",
      "[LOG]Among them, number of categorical feature: 1277\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n",
      "---------------------------------------------\n",
      "fold 4\n",
      "1219 11:44:21\n",
      "[LOG]Reading train&test csv...\n",
      "[LOG]Process df row by row and update dict.\n",
      "[LOG]Reading patient list, LoS, TUD...\n",
      "[LOG]Identify one-hot encoded event, extract 'true event' and value.\n",
      "[LOG]Count occurrence for each event; Remove 'na' in value.\n",
      "[LOG]Event filtering. Remove events that occurred for less than 7 times.\n",
      "[LOG]- Number of events that ever occurred: 1896\n",
      "[LOG]- Number of events that occurred for more than 7 times: 817\n",
      "[LOG]Identify categorical event.\n",
      "[LOG]One-hot encoding categorical event. Scalarize numerical.\n",
      "[LOG]Value filtering. Remove values that occurred for less than 3 times.\n",
      "[LOG]After one-hot encoding, number of feature: 1709\n",
      "[LOG]Among them, number of categorical feature: 1277\n",
      "[LOG]Simplify output dict and check correctness.\n",
      "[LOG]Impute missing value.\n"
     ]
    }
   ],
   "source": [
    "dataset_idxs = [0, 1] # 0: \"pancreatitis\", 1: \"ich\", 2: \"sepsis\"\n",
    "train_dfs, test_dfs, unique_times, dataset_names = load_val_data(dataset_idxs, verbose=True, data_path=\"../../dataset/\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Survival.KNNKaplanMeier import KNNKaplanMeier\n",
    "from Survival.CoxPHModel import CoxPHModel\n",
    "from Survival.AalenAdditiveModel import AalenAdditiveModel\n",
    "from Survival.WeibullRegressionModel import WeibullRegressionModel\n",
    "from Survival.RandomSurvivalForest import RandomSurvivalForest\n",
    "from Survival.NeuralNetworkCox import NeuralNetworkCox\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_collection = {\n",
    "    \"KNN\": KNNKaplanMeier(n_neighbors=5),\n",
    "    \"COX\": CoxPHModel(lambda_=0.04), # CoxPHModel(lambda_=0.12)\n",
    "    \"AAM\": AalenAdditiveModel(coef_penalizer=0.86), # AalenAdditiveModel(coef_penalizer=0.63)\n",
    "    \"WBR\": WeibullRegressionModel(pca_flag=True, n_components=16),\n",
    "    \"RSF\": RandomSurvivalForest(n_trees=50, max_features=5, max_depth=3),\n",
    "    \"NNC\": NeuralNetworkCox(first_layer_size=192, lmbda=0.0)\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model_collection[\"RSF\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model.fit(train_dfs['pancreatitis'][0], duration_col='LOS', event_col='OUT')\n",
    "time_list = unique_times['pancreatitis']\n",
    "median_avg, proba_matrix = model.predict(test_dfs['pancreatitis'][0], time_list=time_list)\n",
    "median_pred = model.pred_median_time(test_dfs['pancreatitis'][0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}