Creating_Features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we combine all the cleaned data to create a master table, on which we train the data for predictive modelling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'EPIC_EMR_cleaned/'\n",
    "information = pd.read_csv(os.path.join(path,'patient_information_cleaned.csv'))\n",
    "information = information.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>HOSP_ADMSN_TIME</th>\n",
       "      <th>HOSP_DISCH_TIME</th>\n",
       "      <th>LOS</th>\n",
       "      <th>SURGERY_DATE</th>\n",
       "      <th>BIRTH_DATE</th>\n",
       "      <th>WEIGHT</th>\n",
       "      <th>IN_OR_DTTM</th>\n",
       "      <th>OUT_OR_DTTM</th>\n",
       "      <th>AN_START_DATETIME</th>\n",
       "      <th>AN_STOP_DATETIME</th>\n",
       "      <th>ICU_ADMIN_FLAG_encoded</th>\n",
       "      <th>SEX_encoded</th>\n",
       "      <th>PRIMARY_ANES_TYPE_NM_encoded</th>\n",
       "      <th>PATIENT_CLASS_GROUP_encoded</th>\n",
       "      <th>DISCH_DISP_encoded</th>\n",
       "      <th>ASA_RATING_encoded</th>\n",
       "      <th>HEIGHT_IN_CM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6af225abbb809cec</td>\n",
       "      <td>7c32ed97b90276e9</td>\n",
       "      <td>10/28/20 9:28</td>\n",
       "      <td>11/5/20 18:52</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10/28/20 0:00</td>\n",
       "      <td>55</td>\n",
       "      <td>2620.83</td>\n",
       "      <td>10/28/20 12:38</td>\n",
       "      <td>10/28/20 16:02</td>\n",
       "      <td>10/28/20 12:38</td>\n",
       "      <td>10/28/20 16:08</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>152.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3119d72120dacb7c</td>\n",
       "      <td>cdc958fff5570ab0</td>\n",
       "      <td>4/7/19 5:18</td>\n",
       "      <td>4/8/19 9:50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4/7/19 0:00</td>\n",
       "      <td>54</td>\n",
       "      <td>2125.23</td>\n",
       "      <td>4/7/19 7:12</td>\n",
       "      <td>4/7/19 13:38</td>\n",
       "      <td>4/7/19 7:12</td>\n",
       "      <td>4/7/19 13:43</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>157.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f0d16bb8e593218e</td>\n",
       "      <td>95d54ee9f5abfab3</td>\n",
       "      <td>12/14/18 7:16</td>\n",
       "      <td>12/16/18 16:05</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12/14/18 0:00</td>\n",
       "      <td>43</td>\n",
       "      <td>3710.78</td>\n",
       "      <td>12/14/18 8:44</td>\n",
       "      <td>12/14/18 12:14</td>\n",
       "      <td>12/14/18 8:46</td>\n",
       "      <td>12/14/18 12:30</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>180.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32f1980e3944170e</td>\n",
       "      <td>0dcace1cd2a4d89d</td>\n",
       "      <td>6/22/19 8:10</td>\n",
       "      <td>6/23/19 11:15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6/22/19 0:00</td>\n",
       "      <td>79</td>\n",
       "      <td>1968.27</td>\n",
       "      <td>6/22/19 10:36</td>\n",
       "      <td>6/22/19 13:05</td>\n",
       "      <td>6/22/19 10:36</td>\n",
       "      <td>6/22/19 13:12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db83846371c937a9</td>\n",
       "      <td>e3025ead155cb9bb</td>\n",
       "      <td>10/12/19 8:43</td>\n",
       "      <td>10/13/19 18:02</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10/12/19 0:00</td>\n",
       "      <td>67</td>\n",
       "      <td>2567.92</td>\n",
       "      <td>10/12/19 10:38</td>\n",
       "      <td>10/12/19 14:31</td>\n",
       "      <td>10/12/19 10:38</td>\n",
       "      <td>10/12/19 14:37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>163.83</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             LOG_ID               MRN HOSP_ADMSN_TIME HOSP_DISCH_TIME  LOS  \\\n",
       "0  6af225abbb809cec  7c32ed97b90276e9   10/28/20 9:28   11/5/20 18:52  8.0   \n",
       "1  3119d72120dacb7c  cdc958fff5570ab0     4/7/19 5:18     4/8/19 9:50  1.0   \n",
       "2  f0d16bb8e593218e  95d54ee9f5abfab3   12/14/18 7:16  12/16/18 16:05  2.0   \n",
       "3  32f1980e3944170e  0dcace1cd2a4d89d    6/22/19 8:10   6/23/19 11:15  1.0   \n",
       "4  db83846371c937a9  e3025ead155cb9bb   10/12/19 8:43  10/13/19 18:02  1.0   \n",
       "\n",
       "    SURGERY_DATE  BIRTH_DATE   WEIGHT      IN_OR_DTTM     OUT_OR_DTTM  \\\n",
       "0  10/28/20 0:00          55  2620.83  10/28/20 12:38  10/28/20 16:02   \n",
       "1    4/7/19 0:00          54  2125.23     4/7/19 7:12    4/7/19 13:38   \n",
       "2  12/14/18 0:00          43  3710.78   12/14/18 8:44  12/14/18 12:14   \n",
       "3   6/22/19 0:00          79  1968.27   6/22/19 10:36   6/22/19 13:05   \n",
       "4  10/12/19 0:00          67  2567.92  10/12/19 10:38  10/12/19 14:31   \n",
       "\n",
       "  AN_START_DATETIME AN_STOP_DATETIME  ICU_ADMIN_FLAG_encoded  SEX_encoded  \\\n",
       "0    10/28/20 12:38   10/28/20 16:08                       0            2   \n",
       "1       4/7/19 7:12     4/7/19 13:43                       0            2   \n",
       "2     12/14/18 8:46   12/14/18 12:30                       1            1   \n",
       "3     6/22/19 10:36    6/22/19 13:12                       1            1   \n",
       "4    10/12/19 10:38   10/12/19 14:37                       1            2   \n",
       "\n",
       "   PRIMARY_ANES_TYPE_NM_encoded  PATIENT_CLASS_GROUP_encoded  \\\n",
       "0                            10                            0   \n",
       "1                             1                            1   \n",
       "2                             1                            1   \n",
       "3                             1                            1   \n",
       "4                             1                            1   \n",
       "\n",
       "   DISCH_DISP_encoded  ASA_RATING_encoded  HEIGHT_IN_CM  \n",
       "0                20.0                 3.0        152.40  \n",
       "1                15.0                 2.0        157.48  \n",
       "2                15.0                 2.0        180.34  \n",
       "3                15.0                 3.0           NaN  \n",
       "4                15.0                 2.0        163.83  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "information.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>HOSP_ADMSN_TIME</th>\n",
       "      <th>HOSP_DISCH_TIME</th>\n",
       "      <th>LOS</th>\n",
       "      <th>SURGERY_DATE</th>\n",
       "      <th>BIRTH_DATE</th>\n",
       "      <th>WEIGHT</th>\n",
       "      <th>IN_OR_DTTM</th>\n",
       "      <th>OUT_OR_DTTM</th>\n",
       "      <th>AN_START_DATETIME</th>\n",
       "      <th>AN_STOP_DATETIME</th>\n",
       "      <th>ICU_ADMIN_FLAG_encoded</th>\n",
       "      <th>SEX_encoded</th>\n",
       "      <th>PRIMARY_ANES_TYPE_NM_encoded</th>\n",
       "      <th>PATIENT_CLASS_GROUP_encoded</th>\n",
       "      <th>DISCH_DISP_encoded</th>\n",
       "      <th>ASA_RATING_encoded</th>\n",
       "      <th>HEIGHT_IN_CM</th>\n",
       "      <th>AN_DURATION</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6af225abbb809cec</td>\n",
       "      <td>7c32ed97b90276e9</td>\n",
       "      <td>10/28/20 9:28</td>\n",
       "      <td>11/5/20 18:52</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10/28/20 0:00</td>\n",
       "      <td>55</td>\n",
       "      <td>2620.83</td>\n",
       "      <td>2020-10-28 12:38:00</td>\n",
       "      <td>10/28/20 16:02</td>\n",
       "      <td>2020-10-28 12:38:00</td>\n",
       "      <td>2020-10-28 16:08:00</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>152.40</td>\n",
       "      <td>3.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3119d72120dacb7c</td>\n",
       "      <td>cdc958fff5570ab0</td>\n",
       "      <td>4/7/19 5:18</td>\n",
       "      <td>4/8/19 9:50</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4/7/19 0:00</td>\n",
       "      <td>54</td>\n",
       "      <td>2125.23</td>\n",
       "      <td>2019-04-07 07:12:00</td>\n",
       "      <td>4/7/19 13:38</td>\n",
       "      <td>2019-04-07 07:12:00</td>\n",
       "      <td>2019-04-07 13:43:00</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>157.48</td>\n",
       "      <td>6.516667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f0d16bb8e593218e</td>\n",
       "      <td>95d54ee9f5abfab3</td>\n",
       "      <td>12/14/18 7:16</td>\n",
       "      <td>12/16/18 16:05</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12/14/18 0:00</td>\n",
       "      <td>43</td>\n",
       "      <td>3710.78</td>\n",
       "      <td>2018-12-14 08:44:00</td>\n",
       "      <td>12/14/18 12:14</td>\n",
       "      <td>2018-12-14 08:46:00</td>\n",
       "      <td>2018-12-14 12:30:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>180.34</td>\n",
       "      <td>3.733333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32f1980e3944170e</td>\n",
       "      <td>0dcace1cd2a4d89d</td>\n",
       "      <td>6/22/19 8:10</td>\n",
       "      <td>6/23/19 11:15</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6/22/19 0:00</td>\n",
       "      <td>79</td>\n",
       "      <td>1968.27</td>\n",
       "      <td>2019-06-22 10:36:00</td>\n",
       "      <td>6/22/19 13:05</td>\n",
       "      <td>2019-06-22 10:36:00</td>\n",
       "      <td>2019-06-22 13:12:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db83846371c937a9</td>\n",
       "      <td>e3025ead155cb9bb</td>\n",
       "      <td>10/12/19 8:43</td>\n",
       "      <td>10/13/19 18:02</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10/12/19 0:00</td>\n",
       "      <td>67</td>\n",
       "      <td>2567.92</td>\n",
       "      <td>2019-10-12 10:38:00</td>\n",
       "      <td>10/12/19 14:31</td>\n",
       "      <td>2019-10-12 10:38:00</td>\n",
       "      <td>2019-10-12 14:37:00</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>163.83</td>\n",
       "      <td>3.983333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             LOG_ID               MRN HOSP_ADMSN_TIME HOSP_DISCH_TIME  LOS  \\\n",
       "0  6af225abbb809cec  7c32ed97b90276e9   10/28/20 9:28   11/5/20 18:52  8.0   \n",
       "1  3119d72120dacb7c  cdc958fff5570ab0     4/7/19 5:18     4/8/19 9:50  1.0   \n",
       "2  f0d16bb8e593218e  95d54ee9f5abfab3   12/14/18 7:16  12/16/18 16:05  2.0   \n",
       "3  32f1980e3944170e  0dcace1cd2a4d89d    6/22/19 8:10   6/23/19 11:15  1.0   \n",
       "4  db83846371c937a9  e3025ead155cb9bb   10/12/19 8:43  10/13/19 18:02  1.0   \n",
       "\n",
       "    SURGERY_DATE  BIRTH_DATE   WEIGHT          IN_OR_DTTM     OUT_OR_DTTM  \\\n",
       "0  10/28/20 0:00          55  2620.83 2020-10-28 12:38:00  10/28/20 16:02   \n",
       "1    4/7/19 0:00          54  2125.23 2019-04-07 07:12:00    4/7/19 13:38   \n",
       "2  12/14/18 0:00          43  3710.78 2018-12-14 08:44:00  12/14/18 12:14   \n",
       "3   6/22/19 0:00          79  1968.27 2019-06-22 10:36:00   6/22/19 13:05   \n",
       "4  10/12/19 0:00          67  2567.92 2019-10-12 10:38:00  10/12/19 14:31   \n",
       "\n",
       "    AN_START_DATETIME    AN_STOP_DATETIME  ICU_ADMIN_FLAG_encoded  \\\n",
       "0 2020-10-28 12:38:00 2020-10-28 16:08:00                       0   \n",
       "1 2019-04-07 07:12:00 2019-04-07 13:43:00                       0   \n",
       "2 2018-12-14 08:46:00 2018-12-14 12:30:00                       1   \n",
       "3 2019-06-22 10:36:00 2019-06-22 13:12:00                       1   \n",
       "4 2019-10-12 10:38:00 2019-10-12 14:37:00                       1   \n",
       "\n",
       "   SEX_encoded  PRIMARY_ANES_TYPE_NM_encoded  PATIENT_CLASS_GROUP_encoded  \\\n",
       "0            2                            10                            0   \n",
       "1            2                             1                            1   \n",
       "2            1                             1                            1   \n",
       "3            1                             1                            1   \n",
       "4            2                             1                            1   \n",
       "\n",
       "   DISCH_DISP_encoded  ASA_RATING_encoded  HEIGHT_IN_CM  AN_DURATION  \n",
       "0                20.0                 3.0        152.40     3.500000  \n",
       "1                15.0                 2.0        157.48     6.516667  \n",
       "2                15.0                 2.0        180.34     3.733333  \n",
       "3                15.0                 3.0           NaN     2.600000  \n",
       "4                15.0                 2.0        163.83     3.983333  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Convert all date columns stored as strings into datetime format\n",
    "#Create a new column for anesthesia duration from AN-Start and AN-Stop\n",
    "information['AN_START_DATETIME'] = pd.to_datetime(information['AN_START_DATETIME'],format='%m/%d/%y %H:%M')\n",
    "information['AN_STOP_DATETIME'] = pd.to_datetime(information['AN_STOP_DATETIME'],format='%m/%d/%y %H:%M')\n",
    "information['IN_OR_DTTM'] = pd.to_datetime(information['IN_OR_DTTM'],format='%m/%d/%y %H:%M')\n",
    "information['AN_DURATION'] = (information['AN_STOP_DATETIME'] - information['AN_START_DATETIME']).dt.seconds / 3600\n",
    "information.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(57959, 3)\n"
     ]
    }
   ],
   "source": [
    "#Remove all null values in IN_OR_DTTM column\n",
    "information = information[information['IN_OR_DTTM'].notna()]\n",
    "log_id_cutoff = information[['LOG_ID','MRN','IN_OR_DTTM']].copy(deep=True)\n",
    "print(log_id_cutoff.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>IN_OR_DTTM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6af225abbb809cec</td>\n",
       "      <td>7c32ed97b90276e9</td>\n",
       "      <td>2020-10-28 12:38:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3119d72120dacb7c</td>\n",
       "      <td>cdc958fff5570ab0</td>\n",
       "      <td>2019-04-07 07:12:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f0d16bb8e593218e</td>\n",
       "      <td>95d54ee9f5abfab3</td>\n",
       "      <td>2018-12-14 08:44:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32f1980e3944170e</td>\n",
       "      <td>0dcace1cd2a4d89d</td>\n",
       "      <td>2019-06-22 10:36:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db83846371c937a9</td>\n",
       "      <td>e3025ead155cb9bb</td>\n",
       "      <td>2019-10-12 10:38:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             LOG_ID               MRN          IN_OR_DTTM\n",
       "0  6af225abbb809cec  7c32ed97b90276e9 2020-10-28 12:38:00\n",
       "1  3119d72120dacb7c  cdc958fff5570ab0 2019-04-07 07:12:00\n",
       "2  f0d16bb8e593218e  95d54ee9f5abfab3 2018-12-14 08:44:00\n",
       "3  32f1980e3944170e  0dcace1cd2a4d89d 2019-06-22 10:36:00\n",
       "4  db83846371c937a9  e3025ead155cb9bb 2019-10-12 10:38:00"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_id_cutoff.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Abnormal Flag_encoded    0\n",
       "Collection Datetime      0\n",
       "LOG_ID                   0\n",
       "MRN                      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'EPIC_EMR_cleaned/'\n",
    "labs = pd.read_csv(os.path.join(path,'patient_labs_cleaned.csv'))\n",
    "labs = labs.drop_duplicates()\n",
    "labs.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Abnormal Flag_encoded</th>\n",
       "      <th>Collection Datetime</th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-10 23:36:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-08 04:09:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-09 05:33:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-12 21:36:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4</td>\n",
       "      <td>2020-03-08 02:36:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Abnormal Flag_encoded  Collection Datetime            LOG_ID  \\\n",
       "0                      1  2020-03-10 23:36:00  3ab8e6344cc9f954   \n",
       "1                      1  2020-03-08 04:09:00  3ab8e6344cc9f954   \n",
       "2                      1  2020-03-09 05:33:00  3ab8e6344cc9f954   \n",
       "3                      1  2020-03-12 21:36:00  3ab8e6344cc9f954   \n",
       "5                      4  2020-03-08 02:36:00  3ab8e6344cc9f954   \n",
       "\n",
       "                MRN  \n",
       "0  559c869f9d7db8ee  \n",
       "1  559c869f9d7db8ee  \n",
       "2  559c869f9d7db8ee  \n",
       "3  559c869f9d7db8ee  \n",
       "5  559c869f9d7db8ee  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Abnormal Flag_encoded</th>\n",
       "      <th>Collection Datetime</th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>IN_OR_DTTM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-10 23:36:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "      <td>2020-03-12 08:04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-08 04:09:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "      <td>2020-03-12 08:04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2020-03-09 05:33:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "      <td>2020-03-12 08:04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2020-03-08 02:36:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "      <td>2020-03-12 08:04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2020-03-07 23:33:00</td>\n",
       "      <td>3ab8e6344cc9f954</td>\n",
       "      <td>559c869f9d7db8ee</td>\n",
       "      <td>2020-03-12 08:04:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Abnormal Flag_encoded Collection Datetime            LOG_ID  \\\n",
       "0                      1 2020-03-10 23:36:00  3ab8e6344cc9f954   \n",
       "1                      1 2020-03-08 04:09:00  3ab8e6344cc9f954   \n",
       "2                      1 2020-03-09 05:33:00  3ab8e6344cc9f954   \n",
       "4                      4 2020-03-08 02:36:00  3ab8e6344cc9f954   \n",
       "5                      5 2020-03-07 23:33:00  3ab8e6344cc9f954   \n",
       "\n",
       "                MRN          IN_OR_DTTM  \n",
       "0  559c869f9d7db8ee 2020-03-12 08:04:00  \n",
       "1  559c869f9d7db8ee 2020-03-12 08:04:00  \n",
       "2  559c869f9d7db8ee 2020-03-12 08:04:00  \n",
       "4  559c869f9d7db8ee 2020-03-12 08:04:00  \n",
       "5  559c869f9d7db8ee 2020-03-12 08:04:00  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labs = labs.merge(log_id_cutoff[['LOG_ID','IN_OR_DTTM']],how='inner',on='LOG_ID')#Join Labs table and LOGS ID table on Log_ID\n",
    "labs['Collection Datetime'] = pd.to_datetime(labs['Collection Datetime'])#Convert Collection Datetime from string to datetime\n",
    "labs = labs[labs['Collection Datetime'] <= labs['IN_OR_DTTM']]\n",
    "labs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store the number of Abnormal flags recorded for each Patient Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 4, 5, 2, 0, 3]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(labs['Abnormal Flag_encoded'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Num_labs_1_count</th>\n",
       "      <th>Num_labs_4_count</th>\n",
       "      <th>Num_labs_5_count</th>\n",
       "      <th>Num_labs_2_count</th>\n",
       "      <th>Num_labs_0_count</th>\n",
       "      <th>Num_labs_3_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3ab8e6344cc9f954</th>\n",
       "      <td>41.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f07e166781abd026</th>\n",
       "      <td>60.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dbb42e6801cf1e62</th>\n",
       "      <td>77.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3a19916d5a0dc3ec</th>\n",
       "      <td>25.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6f0aadfadfce3e20</th>\n",
       "      <td>17.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b31b1841a16cfc5</th>\n",
       "      <td>303.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b40a80aaf894a847</th>\n",
       "      <td>17.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34031ce6a0d152a7</th>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fedac74965a05b3f</th>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ce1f1dd5a63a8f30</th>\n",
       "      <td>15.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Num_labs_1_count  Num_labs_4_count  Num_labs_5_count  \\\n",
       "3ab8e6344cc9f954              41.0              29.0              30.0   \n",
       "f07e166781abd026              60.0              50.0              46.0   \n",
       "dbb42e6801cf1e62              77.0              57.0              83.0   \n",
       "3a19916d5a0dc3ec              25.0              18.0              14.0   \n",
       "6f0aadfadfce3e20              17.0               9.0              11.0   \n",
       "0b31b1841a16cfc5             303.0             242.0             258.0   \n",
       "b40a80aaf894a847              17.0               5.0               7.0   \n",
       "34031ce6a0d152a7               6.0               2.0               3.0   \n",
       "fedac74965a05b3f               9.0               3.0               4.0   \n",
       "ce1f1dd5a63a8f30              15.0               9.0               8.0   \n",
       "\n",
       "                  Num_labs_2_count  Num_labs_0_count  Num_labs_3_count  \n",
       "3ab8e6344cc9f954               6.0               NaN               NaN  \n",
       "f07e166781abd026               1.0              12.0               2.0  \n",
       "dbb42e6801cf1e62               5.0              16.0               4.0  \n",
       "3a19916d5a0dc3ec               1.0               NaN               NaN  \n",
       "6f0aadfadfce3e20               1.0               NaN               NaN  \n",
       "0b31b1841a16cfc5               8.0               8.0               6.0  \n",
       "b40a80aaf894a847               3.0               NaN               NaN  \n",
       "34031ce6a0d152a7               1.0               NaN               1.0  \n",
       "fedac74965a05b3f               2.0               NaN               NaN  \n",
       "ce1f1dd5a63a8f30               NaN               1.0               NaN  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lab_report_df = pd.DataFrame(index=labs['LOG_ID'].unique())\n",
    "for encoding in list(labs['Abnormal Flag_encoded'].unique()):\n",
    "    labs_temp = labs[labs['Abnormal Flag_encoded'] == encoding]\n",
    "    lab_report_df['Num_labs_'+str(encoding)+'_count'] = labs_temp.groupby(by='LOG_ID')['Abnormal Flag_encoded'].count()\n",
    "lab_report_df.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'EPIC_EMR_cleaned/'\n",
    "history = pd.read_csv(os.path.join(path,'patient_history_cleaned.csv'))\n",
    "history = history.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mrn</th>\n",
       "      <th>dx_name_encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cdce1f368362568f</td>\n",
       "      <td>32990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>309411c5b6b9658e</td>\n",
       "      <td>45319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13e76a6d564f8049</td>\n",
       "      <td>37245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f42b6b0d38e95a7d</td>\n",
       "      <td>37245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ef7b3e727d8cc736</td>\n",
       "      <td>37245</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                mrn  dx_name_encoded\n",
       "0  cdce1f368362568f            32990\n",
       "1  309411c5b6b9658e            45319\n",
       "2  13e76a6d564f8049            37245\n",
       "3  f42b6b0d38e95a7d            37245\n",
       "4  ef7b3e727d8cc736            37245"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "history.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MRN</th>\n",
       "      <th>Num_Past_Visits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>121590d818fc810e</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>477e792e1ae54cf2</td>\n",
       "      <td>169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e63a4d4458fca74a</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cb0719ad7fd07fcf</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a5b4f6c2312d4fa</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                MRN  Num_Past_Visits\n",
       "0  121590d818fc810e              175\n",
       "1  477e792e1ae54cf2              169\n",
       "2  e63a4d4458fca74a              163\n",
       "3  cb0719ad7fd07fcf              155\n",
       "4  5a5b4f6c2312d4fa              142"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Store the number of visits of each patients\n",
    "num_of_past_visits = history.groupby('mrn')['dx_name_encoded'].count().sort_values(ascending=False).reset_index()\n",
    "num_of_past_visits = num_of_past_visits.rename(columns={\"mrn\":'MRN','dx_name_encoded':'Num_Past_Visits'})\n",
    "num_of_past_visits.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'EPIC_EMR_cleaned/'\n",
    "procedure_event = pd.read_csv(os.path.join(path,'patient_procedure_events_cleaned.csv'))\n",
    "procedure_event = procedure_event.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>EVENT_TIME</th>\n",
       "      <th>EVENT_DISPLAY_NAME_encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e1a066d449ed18b0</td>\n",
       "      <td>499386c1a8693fd0</td>\n",
       "      <td>4/27/19 3:04</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>068b548cf0d91976</td>\n",
       "      <td>68d58e4c2dcd06f6</td>\n",
       "      <td>5/6/19 11:46</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05195f764521086f</td>\n",
       "      <td>85a062b39530bf42</td>\n",
       "      <td>6/12/19 20:59</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>33a907e9affc529f</td>\n",
       "      <td>f7bf5c74f687a530</td>\n",
       "      <td>9/23/19 7:57</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59dc47ed0b52c5d8</td>\n",
       "      <td>424d7ceb87a7defd</td>\n",
       "      <td>9/3/19 7:24</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             LOG_ID               MRN     EVENT_TIME  \\\n",
       "0  e1a066d449ed18b0  499386c1a8693fd0   4/27/19 3:04   \n",
       "1  068b548cf0d91976  68d58e4c2dcd06f6   5/6/19 11:46   \n",
       "2  05195f764521086f  85a062b39530bf42  6/12/19 20:59   \n",
       "3  33a907e9affc529f  f7bf5c74f687a530   9/23/19 7:57   \n",
       "4  59dc47ed0b52c5d8  424d7ceb87a7defd    9/3/19 7:24   \n",
       "\n",
       "   EVENT_DISPLAY_NAME_encoded  \n",
       "0                          72  \n",
       "1                          72  \n",
       "2                          27  \n",
       "3                          55  \n",
       "4                          29  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "procedure_event.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "procedure_event = procedure_event.merge(log_id_cutoff[['LOG_ID','IN_OR_DTTM']],how='inner',on='LOG_ID')\n",
    "procedure_event['EVENT_TIME'] = pd.to_datetime(procedure_event['EVENT_TIME'],format='%m/%d/%y %H:%M')\n",
    "procedure_event = procedure_event[procedure_event['EVENT_TIME'] <= procedure_event['IN_OR_DTTM']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store the number of times each procedure is performed for each Patient Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "procedure_features = pd.DataFrame(index=procedure_event['LOG_ID'].unique())\n",
    "for encoding in list(procedure_event['EVENT_DISPLAY_NAME_encoded'].unique()):\n",
    "    procedure_temp = procedure_event[procedure_event['EVENT_DISPLAY_NAME_encoded'] == encoding]\n",
    "    procedure_features['Num_procedure_'+str(encoding)+'_count'] = procedure_temp.groupby(by='LOG_ID')['EVENT_DISPLAY_NAME_encoded'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Num_procedure_85_count</th>\n",
       "      <th>Num_procedure_49_count</th>\n",
       "      <th>Num_procedure_89_count</th>\n",
       "      <th>Num_procedure_55_count</th>\n",
       "      <th>Num_procedure_72_count</th>\n",
       "      <th>Num_procedure_79_count</th>\n",
       "      <th>Num_procedure_50_count</th>\n",
       "      <th>Num_procedure_21_count</th>\n",
       "      <th>Num_procedure_88_count</th>\n",
       "      <th>Num_procedure_71_count</th>\n",
       "      <th>...</th>\n",
       "      <th>Num_procedure_44_count</th>\n",
       "      <th>Num_procedure_84_count</th>\n",
       "      <th>Num_procedure_30_count</th>\n",
       "      <th>Num_procedure_63_count</th>\n",
       "      <th>Num_procedure_32_count</th>\n",
       "      <th>Num_procedure_9_count</th>\n",
       "      <th>Num_procedure_10_count</th>\n",
       "      <th>Num_procedure_80_count</th>\n",
       "      <th>Num_procedure_8_count</th>\n",
       "      <th>Num_procedure_53_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>e1a066d449ed18b0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>068b548cf0d91976</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>05195f764521086f</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33a907e9affc529f</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59dc47ed0b52c5d8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Num_procedure_85_count  Num_procedure_49_count  \\\n",
       "e1a066d449ed18b0                     1.0                     1.0   \n",
       "068b548cf0d91976                     1.0                     NaN   \n",
       "05195f764521086f                     1.0                     1.0   \n",
       "33a907e9affc529f                     1.0                     1.0   \n",
       "59dc47ed0b52c5d8                     1.0                     NaN   \n",
       "\n",
       "                  Num_procedure_89_count  Num_procedure_55_count  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN   \n",
       "068b548cf0d91976                     NaN                     NaN   \n",
       "05195f764521086f                     NaN                     NaN   \n",
       "33a907e9affc529f                     NaN                     NaN   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN   \n",
       "\n",
       "                  Num_procedure_72_count  Num_procedure_79_count  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN   \n",
       "068b548cf0d91976                     NaN                     NaN   \n",
       "05195f764521086f                     NaN                     NaN   \n",
       "33a907e9affc529f                     NaN                     NaN   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN   \n",
       "\n",
       "                  Num_procedure_50_count  Num_procedure_21_count  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN   \n",
       "068b548cf0d91976                     NaN                     NaN   \n",
       "05195f764521086f                     NaN                     NaN   \n",
       "33a907e9affc529f                     NaN                     NaN   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN   \n",
       "\n",
       "                  Num_procedure_88_count  Num_procedure_71_count  ...  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN  ...   \n",
       "068b548cf0d91976                     NaN                     NaN  ...   \n",
       "05195f764521086f                     NaN                     NaN  ...   \n",
       "33a907e9affc529f                     NaN                     NaN  ...   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN  ...   \n",
       "\n",
       "                  Num_procedure_44_count  Num_procedure_84_count  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN   \n",
       "068b548cf0d91976                     NaN                     NaN   \n",
       "05195f764521086f                     NaN                     NaN   \n",
       "33a907e9affc529f                     NaN                     NaN   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN   \n",
       "\n",
       "                  Num_procedure_30_count  Num_procedure_63_count  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN   \n",
       "068b548cf0d91976                     NaN                     NaN   \n",
       "05195f764521086f                     NaN                     NaN   \n",
       "33a907e9affc529f                     NaN                     NaN   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN   \n",
       "\n",
       "                  Num_procedure_32_count  Num_procedure_9_count  \\\n",
       "e1a066d449ed18b0                     NaN                    NaN   \n",
       "068b548cf0d91976                     NaN                    NaN   \n",
       "05195f764521086f                     NaN                    NaN   \n",
       "33a907e9affc529f                     NaN                    NaN   \n",
       "59dc47ed0b52c5d8                     NaN                    NaN   \n",
       "\n",
       "                  Num_procedure_10_count  Num_procedure_80_count  \\\n",
       "e1a066d449ed18b0                     NaN                     NaN   \n",
       "068b548cf0d91976                     NaN                     NaN   \n",
       "05195f764521086f                     NaN                     NaN   \n",
       "33a907e9affc529f                     NaN                     NaN   \n",
       "59dc47ed0b52c5d8                     NaN                     NaN   \n",
       "\n",
       "                  Num_procedure_8_count  Num_procedure_53_count  \n",
       "e1a066d449ed18b0                    NaN                     NaN  \n",
       "068b548cf0d91976                    NaN                     NaN  \n",
       "05195f764521086f                    NaN                     NaN  \n",
       "33a907e9affc529f                    NaN                     NaN  \n",
       "59dc47ed0b52c5d8                    NaN                     NaN  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "procedure_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'EPIC_EMR_cleaned/'\n",
    "post_op = pd.read_csv(os.path.join(path,'patient_post_op_complication_cleaned.csv'))\n",
    "post_op = post_op.drop(columns=['SMRTDTA_ELEM_VALUE_encoded','CONTEXT_NAME_encoded'])\n",
    "post_op = post_op.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Element_abbr_encoded</th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>a8b3f9afa63c96d1</td>\n",
       "      <td>aaebaad9587efc99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>5dc0d50fa1d11ea2</td>\n",
       "      <td>dc6e29201905577b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>c66c0c773c2280d8</td>\n",
       "      <td>66d9279ff5f4b927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>bec65b5c7ae241b8</td>\n",
       "      <td>c80dbde76c38ecad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>104d64f326946e24</td>\n",
       "      <td>bb42a6512d570bbc</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Element_abbr_encoded            LOG_ID               MRN\n",
       "0                     2  a8b3f9afa63c96d1  aaebaad9587efc99\n",
       "1                     2  5dc0d50fa1d11ea2  dc6e29201905577b\n",
       "2                     2  c66c0c773c2280d8  66d9279ff5f4b927\n",
       "3                     2  bec65b5c7ae241b8  c80dbde76c38ecad\n",
       "4                     2  104d64f326946e24  bb42a6512d570bbc"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_op.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store the number of times each post operative complication recorded for each Patient Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "post_op_features = pd.DataFrame(index=post_op['LOG_ID'].unique())\n",
    "for encoding in list(post_op['Element_abbr_encoded'].unique()):\n",
    "    post_op_temp = post_op[post_op['Element_abbr_encoded'] == encoding]\n",
    "    post_op_features['Num_Post_OP_'+str(encoding)+'_count'] = post_op_temp.groupby(by='LOG_ID')['Element_abbr_encoded'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Num_Post_OP_2_count</th>\n",
       "      <th>Num_Post_OP_11_count</th>\n",
       "      <th>Num_Post_OP_9_count</th>\n",
       "      <th>Num_Post_OP_7_count</th>\n",
       "      <th>Num_Post_OP_6_count</th>\n",
       "      <th>Num_Post_OP_3_count</th>\n",
       "      <th>Num_Post_OP_0_count</th>\n",
       "      <th>Num_Post_OP_4_count</th>\n",
       "      <th>Num_Post_OP_1_count</th>\n",
       "      <th>Num_Post_OP_5_count</th>\n",
       "      <th>Num_Post_OP_8_count</th>\n",
       "      <th>Num_Post_OP_10_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a8b3f9afa63c96d1</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5dc0d50fa1d11ea2</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c66c0c773c2280d8</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bec65b5c7ae241b8</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104d64f326946e24</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Num_Post_OP_2_count  Num_Post_OP_11_count  \\\n",
       "a8b3f9afa63c96d1                    1                   NaN   \n",
       "5dc0d50fa1d11ea2                    1                   NaN   \n",
       "c66c0c773c2280d8                    1                   NaN   \n",
       "bec65b5c7ae241b8                    1                   NaN   \n",
       "104d64f326946e24                    1                   NaN   \n",
       "\n",
       "                  Num_Post_OP_9_count  Num_Post_OP_7_count  \\\n",
       "a8b3f9afa63c96d1                  NaN                  NaN   \n",
       "5dc0d50fa1d11ea2                  NaN                  NaN   \n",
       "c66c0c773c2280d8                  NaN                  NaN   \n",
       "bec65b5c7ae241b8                  NaN                  1.0   \n",
       "104d64f326946e24                  NaN                  NaN   \n",
       "\n",
       "                  Num_Post_OP_6_count  Num_Post_OP_3_count  \\\n",
       "a8b3f9afa63c96d1                  NaN                  NaN   \n",
       "5dc0d50fa1d11ea2                  NaN                  NaN   \n",
       "c66c0c773c2280d8                  NaN                  NaN   \n",
       "bec65b5c7ae241b8                  NaN                  1.0   \n",
       "104d64f326946e24                  NaN                  NaN   \n",
       "\n",
       "                  Num_Post_OP_0_count  Num_Post_OP_4_count  \\\n",
       "a8b3f9afa63c96d1                  NaN                  NaN   \n",
       "5dc0d50fa1d11ea2                  NaN                  NaN   \n",
       "c66c0c773c2280d8                  NaN                  NaN   \n",
       "bec65b5c7ae241b8                  NaN                  NaN   \n",
       "104d64f326946e24                  NaN                  NaN   \n",
       "\n",
       "                  Num_Post_OP_1_count  Num_Post_OP_5_count  \\\n",
       "a8b3f9afa63c96d1                  NaN                  NaN   \n",
       "5dc0d50fa1d11ea2                  NaN                  NaN   \n",
       "c66c0c773c2280d8                  NaN                  NaN   \n",
       "bec65b5c7ae241b8                  NaN                  NaN   \n",
       "104d64f326946e24                  NaN                  NaN   \n",
       "\n",
       "                  Num_Post_OP_8_count  Num_Post_OP_10_count  \n",
       "a8b3f9afa63c96d1                  NaN                   NaN  \n",
       "5dc0d50fa1d11ea2                  NaN                   NaN  \n",
       "c66c0c773c2280d8                  NaN                   NaN  \n",
       "bec65b5c7ae241b8                  NaN                   NaN  \n",
       "104d64f326946e24                  NaN                   NaN  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "post_op_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(57952, 11)\n"
     ]
    }
   ],
   "source": [
    "final_dataset = information[['LOG_ID',\"MRN\",'LOS','BIRTH_DATE','WEIGHT','ICU_ADMIN_FLAG_encoded','SEX_encoded',\n",
    "'PRIMARY_ANES_TYPE_NM_encoded','PATIENT_CLASS_GROUP_encoded','ASA_RATING_encoded','HEIGHT_IN_CM',]].copy(deep=True)# remove an duration to prevent data leakage\n",
    "final_dataset = final_dataset.drop_duplicates(subset=['LOG_ID'],keep='first')\n",
    "print(final_dataset.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>LOS</th>\n",
       "      <th>BIRTH_DATE</th>\n",
       "      <th>WEIGHT</th>\n",
       "      <th>ICU_ADMIN_FLAG_encoded</th>\n",
       "      <th>SEX_encoded</th>\n",
       "      <th>PRIMARY_ANES_TYPE_NM_encoded</th>\n",
       "      <th>PATIENT_CLASS_GROUP_encoded</th>\n",
       "      <th>ASA_RATING_encoded</th>\n",
       "      <th>HEIGHT_IN_CM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6af225abbb809cec</td>\n",
       "      <td>7c32ed97b90276e9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>55</td>\n",
       "      <td>2620.83</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>152.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3119d72120dacb7c</td>\n",
       "      <td>cdc958fff5570ab0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54</td>\n",
       "      <td>2125.23</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>157.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f0d16bb8e593218e</td>\n",
       "      <td>95d54ee9f5abfab3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43</td>\n",
       "      <td>3710.78</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>180.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32f1980e3944170e</td>\n",
       "      <td>0dcace1cd2a4d89d</td>\n",
       "      <td>1.0</td>\n",
       "      <td>79</td>\n",
       "      <td>1968.27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db83846371c937a9</td>\n",
       "      <td>e3025ead155cb9bb</td>\n",
       "      <td>1.0</td>\n",
       "      <td>67</td>\n",
       "      <td>2567.92</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>163.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f0aafb8dcb6d1949</td>\n",
       "      <td>e177fdf1ed283ffe</td>\n",
       "      <td>0.0</td>\n",
       "      <td>51</td>\n",
       "      <td>1922.41</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>aea32165b53d1c29</td>\n",
       "      <td>1f9ced1dbc1f9ed4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>47</td>\n",
       "      <td>5114.67</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>172.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>e8aba3cde5cddc4b</td>\n",
       "      <td>50baf65ac33dac67</td>\n",
       "      <td>0.0</td>\n",
       "      <td>78</td>\n",
       "      <td>2528.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>180.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>b52faa639b25b425</td>\n",
       "      <td>b3796981839ce195</td>\n",
       "      <td>1.0</td>\n",
       "      <td>68</td>\n",
       "      <td>3232.83</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>167.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>74fd8b1a2ee73ae3</td>\n",
       "      <td>85bb7b2dfac56ca7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>57</td>\n",
       "      <td>3116.42</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             LOG_ID               MRN  LOS  BIRTH_DATE   WEIGHT  \\\n",
       "0  6af225abbb809cec  7c32ed97b90276e9  8.0          55  2620.83   \n",
       "1  3119d72120dacb7c  cdc958fff5570ab0  1.0          54  2125.23   \n",
       "2  f0d16bb8e593218e  95d54ee9f5abfab3  2.0          43  3710.78   \n",
       "3  32f1980e3944170e  0dcace1cd2a4d89d  1.0          79  1968.27   \n",
       "4  db83846371c937a9  e3025ead155cb9bb  1.0          67  2567.92   \n",
       "5  f0aafb8dcb6d1949  e177fdf1ed283ffe  0.0          51  1922.41   \n",
       "6  aea32165b53d1c29  1f9ced1dbc1f9ed4  0.0          47  5114.67   \n",
       "7  e8aba3cde5cddc4b  50baf65ac33dac67  0.0          78  2528.00   \n",
       "8  b52faa639b25b425  b3796981839ce195  1.0          68  3232.83   \n",
       "9  74fd8b1a2ee73ae3  85bb7b2dfac56ca7  6.0          57  3116.42   \n",
       "\n",
       "   ICU_ADMIN_FLAG_encoded  SEX_encoded  PRIMARY_ANES_TYPE_NM_encoded  \\\n",
       "0                       0            2                            10   \n",
       "1                       0            2                             1   \n",
       "2                       1            1                             1   \n",
       "3                       1            1                             1   \n",
       "4                       1            2                             1   \n",
       "5                       1            2                             1   \n",
       "6                       1            2                             3   \n",
       "7                       1            1                             6   \n",
       "8                       1            1                             1   \n",
       "9                       0            2                             1   \n",
       "\n",
       "   PATIENT_CLASS_GROUP_encoded  ASA_RATING_encoded  HEIGHT_IN_CM  \n",
       "0                            0                 3.0        152.40  \n",
       "1                            1                 2.0        157.48  \n",
       "2                            1                 2.0        180.34  \n",
       "3                            1                 3.0           NaN  \n",
       "4                            1                 2.0        163.83  \n",
       "5                            1                 2.0           NaN  \n",
       "6                            1                 3.0        172.72  \n",
       "7                            1                 NaN        180.34  \n",
       "8                            1                 3.0        167.64  \n",
       "9                            0                 3.0           NaN  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(57952, 59)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset = final_dataset.merge(lab_report_df,right_index=True,left_on='LOG_ID',how='left')\n",
    "final_dataset = final_dataset.merge(num_of_past_visits,on='MRN',how='left')\n",
    "final_dataset = final_dataset.merge(procedure_features,right_index=True,left_on='LOG_ID',how='left')\n",
    "#final_dataset = final_dataset.merge(post_op_features,right_index=True,left_on='LOG_ID',how='left')\n",
    "final_dataset.shape #Avoiding post opeartive features because it is an event after the surgery."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LOG_ID</th>\n",
       "      <th>MRN</th>\n",
       "      <th>LOS</th>\n",
       "      <th>BIRTH_DATE</th>\n",
       "      <th>WEIGHT</th>\n",
       "      <th>ICU_ADMIN_FLAG_encoded</th>\n",
       "      <th>SEX_encoded</th>\n",
       "      <th>PRIMARY_ANES_TYPE_NM_encoded</th>\n",
       "      <th>PATIENT_CLASS_GROUP_encoded</th>\n",
       "      <th>ASA_RATING_encoded</th>\n",
       "      <th>...</th>\n",
       "      <th>Num_procedure_44_count</th>\n",
       "      <th>Num_procedure_84_count</th>\n",
       "      <th>Num_procedure_30_count</th>\n",
       "      <th>Num_procedure_63_count</th>\n",
       "      <th>Num_procedure_32_count</th>\n",
       "      <th>Num_procedure_9_count</th>\n",
       "      <th>Num_procedure_10_count</th>\n",
       "      <th>Num_procedure_80_count</th>\n",
       "      <th>Num_procedure_8_count</th>\n",
       "      <th>Num_procedure_53_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6af225abbb809cec</td>\n",
       "      <td>7c32ed97b90276e9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>55</td>\n",
       "      <td>2620.83</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3119d72120dacb7c</td>\n",
       "      <td>cdc958fff5570ab0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54</td>\n",
       "      <td>2125.23</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f0d16bb8e593218e</td>\n",
       "      <td>95d54ee9f5abfab3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43</td>\n",
       "      <td>3710.78</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32f1980e3944170e</td>\n",
       "      <td>0dcace1cd2a4d89d</td>\n",
       "      <td>1.0</td>\n",
       "      <td>79</td>\n",
       "      <td>1968.27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db83846371c937a9</td>\n",
       "      <td>e3025ead155cb9bb</td>\n",
       "      <td>1.0</td>\n",
       "      <td>67</td>\n",
       "      <td>2567.92</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f0aafb8dcb6d1949</td>\n",
       "      <td>e177fdf1ed283ffe</td>\n",
       "      <td>0.0</td>\n",
       "      <td>51</td>\n",
       "      <td>1922.41</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>aea32165b53d1c29</td>\n",
       "      <td>1f9ced1dbc1f9ed4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>47</td>\n",
       "      <td>5114.67</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>e8aba3cde5cddc4b</td>\n",
       "      <td>50baf65ac33dac67</td>\n",
       "      <td>0.0</td>\n",
       "      <td>78</td>\n",
       "      <td>2528.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>b52faa639b25b425</td>\n",
       "      <td>b3796981839ce195</td>\n",
       "      <td>1.0</td>\n",
       "      <td>68</td>\n",
       "      <td>3232.83</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>74fd8b1a2ee73ae3</td>\n",
       "      <td>85bb7b2dfac56ca7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>57</td>\n",
       "      <td>3116.42</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             LOG_ID               MRN  LOS  BIRTH_DATE   WEIGHT  \\\n",
       "0  6af225abbb809cec  7c32ed97b90276e9  8.0          55  2620.83   \n",
       "1  3119d72120dacb7c  cdc958fff5570ab0  1.0          54  2125.23   \n",
       "2  f0d16bb8e593218e  95d54ee9f5abfab3  2.0          43  3710.78   \n",
       "3  32f1980e3944170e  0dcace1cd2a4d89d  1.0          79  1968.27   \n",
       "4  db83846371c937a9  e3025ead155cb9bb  1.0          67  2567.92   \n",
       "5  f0aafb8dcb6d1949  e177fdf1ed283ffe  0.0          51  1922.41   \n",
       "6  aea32165b53d1c29  1f9ced1dbc1f9ed4  0.0          47  5114.67   \n",
       "7  e8aba3cde5cddc4b  50baf65ac33dac67  0.0          78  2528.00   \n",
       "8  b52faa639b25b425  b3796981839ce195  1.0          68  3232.83   \n",
       "9  74fd8b1a2ee73ae3  85bb7b2dfac56ca7  6.0          57  3116.42   \n",
       "\n",
       "   ICU_ADMIN_FLAG_encoded  SEX_encoded  PRIMARY_ANES_TYPE_NM_encoded  \\\n",
       "0                       0            2                            10   \n",
       "1                       0            2                             1   \n",
       "2                       1            1                             1   \n",
       "3                       1            1                             1   \n",
       "4                       1            2                             1   \n",
       "5                       1            2                             1   \n",
       "6                       1            2                             3   \n",
       "7                       1            1                             6   \n",
       "8                       1            1                             1   \n",
       "9                       0            2                             1   \n",
       "\n",
       "   PATIENT_CLASS_GROUP_encoded  ASA_RATING_encoded  ...  \\\n",
       "0                            0                 3.0  ...   \n",
       "1                            1                 2.0  ...   \n",
       "2                            1                 2.0  ...   \n",
       "3                            1                 3.0  ...   \n",
       "4                            1                 2.0  ...   \n",
       "5                            1                 2.0  ...   \n",
       "6                            1                 3.0  ...   \n",
       "7                            1                 NaN  ...   \n",
       "8                            1                 3.0  ...   \n",
       "9                            0                 3.0  ...   \n",
       "\n",
       "   Num_procedure_44_count  Num_procedure_84_count  Num_procedure_30_count  \\\n",
       "0                     NaN                     NaN                     NaN   \n",
       "1                     NaN                     NaN                     NaN   \n",
       "2                     NaN                     NaN                     NaN   \n",
       "3                     NaN                     NaN                     NaN   \n",
       "4                     NaN                     NaN                     NaN   \n",
       "5                     NaN                     NaN                     NaN   \n",
       "6                     NaN                     NaN                     NaN   \n",
       "7                     NaN                     NaN                     NaN   \n",
       "8                     NaN                     NaN                     NaN   \n",
       "9                     NaN                     NaN                     NaN   \n",
       "\n",
       "   Num_procedure_63_count  Num_procedure_32_count  Num_procedure_9_count  \\\n",
       "0                     NaN                     NaN                    NaN   \n",
       "1                     NaN                     NaN                    NaN   \n",
       "2                     NaN                     NaN                    NaN   \n",
       "3                     NaN                     NaN                    NaN   \n",
       "4                     NaN                     NaN                    NaN   \n",
       "5                     NaN                     NaN                    NaN   \n",
       "6                     NaN                     NaN                    NaN   \n",
       "7                     NaN                     NaN                    NaN   \n",
       "8                     NaN                     NaN                    NaN   \n",
       "9                     NaN                     NaN                    NaN   \n",
       "\n",
       "   Num_procedure_10_count  Num_procedure_80_count  Num_procedure_8_count  \\\n",
       "0                     NaN                     NaN                    NaN   \n",
       "1                     NaN                     NaN                    NaN   \n",
       "2                     NaN                     NaN                    NaN   \n",
       "3                     NaN                     NaN                    NaN   \n",
       "4                     NaN                     NaN                    NaN   \n",
       "5                     NaN                     NaN                    NaN   \n",
       "6                     NaN                     NaN                    NaN   \n",
       "7                     NaN                     NaN                    NaN   \n",
       "8                     NaN                     NaN                    NaN   \n",
       "9                     NaN                     NaN                    NaN   \n",
       "\n",
       "   Num_procedure_53_count  \n",
       "0                     NaN  \n",
       "1                     NaN  \n",
       "2                     NaN  \n",
       "3                     NaN  \n",
       "4                     NaN  \n",
       "5                     NaN  \n",
       "6                     NaN  \n",
       "7                     NaN  \n",
       "8                     NaN  \n",
       "9                     NaN  \n",
       "\n",
       "[10 rows x 59 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataset.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset.to_feather('final_dataset.feather')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}