cleaning_EMR_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import os\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_column_and_replace(df, col_name,encoding_dict_=None):\n",
    "\n",
    "    if encoding_dict_ is None:\n",
    "        out = df.select(pl.col(col_name).unique()).with_row_count()\n",
    "        out = out.select([col_name,'row_nr'])\n",
    "        encoding_dict = dict(out.iter_rows())\n",
    "    else:\n",
    "        encoding_dict = encoding_dict_\n",
    "    df = df.with_columns(pl.col(col_name).map_dict(encoding_dict,return_dtype=pl.UInt64).alias(col_name+'_encoded'))\n",
    "    return df,encoding_dict\n",
    "\n",
    "def encode_two_columns(df, col_1,col_2):\n",
    "    encoding_dict = dict(df.select([col_1,col_2]).unique().iter_rows())\n",
    "    return encoding_dict\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning patient history sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_history_encoding = {}\n",
    "cols_to_encode = ['dx_name']\n",
    "data = pl.read_csv('EPIC_EMR/patient_history.csv',dtypes={'diagnosis_code':pl.Utf8})\n",
    "data = data.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(437721, 3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>mrn</th><th>diagnosis_code</th><th>dx_name</th></tr><tr><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;cdce1f36836256…</td><td>&quot;117.5&quot;</td><td>&quot;Cryptococcosis…</td></tr><tr><td>&quot;309411c5b6b965…</td><td>&quot;153.1&quot;</td><td>&quot;Malignant neop…</td></tr><tr><td>&quot;13e76a6d564f80…</td><td>&quot;238.2&quot;</td><td>&quot;Neoplasm of un…</td></tr><tr><td>&quot;f42b6b0d38e95a…</td><td>&quot;238.2&quot;</td><td>&quot;Neoplasm of un…</td></tr><tr><td>&quot;ef7b3e727d8cc7…</td><td>&quot;238.2&quot;</td><td>&quot;Neoplasm of un…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 3)\n",
       "┌──────────────────┬────────────────┬───────────────────────────────────┐\n",
       "│ mrn              ┆ diagnosis_code ┆ dx_name                           │\n",
       "│ ---              ┆ ---            ┆ ---                               │\n",
       "│ str              ┆ str            ┆ str                               │\n",
       "╞══════════════════╪════════════════╪═══════════════════════════════════╡\n",
       "│ cdce1f368362568f ┆ 117.5          ┆ Cryptococcosis (CMS-HCC)          │\n",
       "│ 309411c5b6b9658e ┆ 153.1          ┆ Malignant neoplasm of transverse… │\n",
       "│ 13e76a6d564f8049 ┆ 238.2          ┆ Neoplasm of uncertain behavior o… │\n",
       "│ f42b6b0d38e95a7d ┆ 238.2          ┆ Neoplasm of uncertain behavior o… │\n",
       "│ ef7b3e727d8cc736 ┆ 238.2          ┆ Neoplasm of uncertain behavior o… │\n",
       "└──────────────────┴────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_history_encoding['dx_name___diagnosis_code'] = encode_two_columns(data,'dx_name','diagnosis_code')\n",
    "for col in cols_to_encode:\n",
    "    data, patient_history_encoding[col] = encode_column_and_replace(data,col)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>mrn</th><th>diagnosis_code</th><th>dx_name</th><th>dx_name_encoded</th></tr><tr><td>str</td><td>str</td><td>str</td><td>u64</td></tr></thead><tbody><tr><td>&quot;cdce1f36836256…</td><td>&quot;117.5&quot;</td><td>&quot;Cryptococcosis…</td><td>32990</td></tr><tr><td>&quot;309411c5b6b965…</td><td>&quot;153.1&quot;</td><td>&quot;Malignant neop…</td><td>45319</td></tr><tr><td>&quot;13e76a6d564f80…</td><td>&quot;238.2&quot;</td><td>&quot;Neoplasm of un…</td><td>37245</td></tr><tr><td>&quot;f42b6b0d38e95a…</td><td>&quot;238.2&quot;</td><td>&quot;Neoplasm of un…</td><td>37245</td></tr><tr><td>&quot;ef7b3e727d8cc7…</td><td>&quot;238.2&quot;</td><td>&quot;Neoplasm of un…</td><td>37245</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌──────────────────┬────────────────┬───────────────────────────────────┬─────────────────┐\n",
       "│ mrn              ┆ diagnosis_code ┆ dx_name                           ┆ dx_name_encoded │\n",
       "│ ---              ┆ ---            ┆ ---                               ┆ ---             │\n",
       "│ str              ┆ str            ┆ str                               ┆ u64             │\n",
       "╞══════════════════╪════════════════╪═══════════════════════════════════╪═════════════════╡\n",
       "│ cdce1f368362568f ┆ 117.5          ┆ Cryptococcosis (CMS-HCC)          ┆ 32990           │\n",
       "│ 309411c5b6b9658e ┆ 153.1          ┆ Malignant neoplasm of transverse… ┆ 45319           │\n",
       "│ 13e76a6d564f8049 ┆ 238.2          ┆ Neoplasm of uncertain behavior o… ┆ 37245           │\n",
       "│ f42b6b0d38e95a7d ┆ 238.2          ┆ Neoplasm of uncertain behavior o… ┆ 37245           │\n",
       "│ ef7b3e727d8cc736 ┆ 238.2          ┆ Neoplasm of uncertain behavior o… ┆ 37245           │\n",
       "└──────────────────┴────────────────┴───────────────────────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>mrn</th><th>diagnosis_code</th><th>dx_name</th><th>dx_name_encoded</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>107424</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 4)\n",
       "┌─────┬────────────────┬─────────┬─────────────────┐\n",
       "│ mrn ┆ diagnosis_code ┆ dx_name ┆ dx_name_encoded │\n",
       "│ --- ┆ ---            ┆ ---     ┆ ---             │\n",
       "│ u32 ┆ u32            ┆ u32     ┆ u32             │\n",
       "╞═════╪════════════════╪═════════╪═════════════════╡\n",
       "│ 0   ┆ 107424         ┆ 0       ┆ 0               │\n",
       "└─────┴────────────────┴─────────┴─────────────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.null_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['dx_name___diagnosis_code', 'dx_name'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "patient_history_encoding.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(437721, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.select([\n",
    "    'mrn','dx_name_encoded'])\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>mrn</th><th>dx_name_encoded</th></tr><tr><td>str</td><td>u64</td></tr></thead><tbody><tr><td>&quot;cdce1f36836256…</td><td>32990</td></tr><tr><td>&quot;309411c5b6b965…</td><td>45319</td></tr><tr><td>&quot;13e76a6d564f80…</td><td>37245</td></tr><tr><td>&quot;f42b6b0d38e95a…</td><td>37245</td></tr><tr><td>&quot;ef7b3e727d8cc7…</td><td>37245</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌──────────────────┬─────────────────┐\n",
       "│ mrn              ┆ dx_name_encoded │\n",
       "│ ---              ┆ ---             │\n",
       "│ str              ┆ u64             │\n",
       "╞══════════════════╪═════════════════╡\n",
       "│ cdce1f368362568f ┆ 32990           │\n",
       "│ 309411c5b6b9658e ┆ 45319           │\n",
       "│ 13e76a6d564f8049 ┆ 37245           │\n",
       "│ f42b6b0d38e95a7d ┆ 37245           │\n",
       "│ ef7b3e727d8cc736 ┆ 37245           │\n",
       "└──────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.write_csv('EPIC_EMR_cleaned/patient_history_cleaned.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59056"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(patient_history_encoding['dx_name'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning patient information\n",
    "Here The columns ICU_ADMIN_FLAG, SEX, PRIMARY_ANES_TYPE_NM, PATIENT_CLASS_GROUP, DISCH_DISP_C, ASA_RATING_C are encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_information_encoding = {}\n",
    "cols_to_encode = ['ICU_ADMIN_FLAG','SEX','PRIMARY_ANES_TYPE_NM','PATIENT_CLASS_GROUP'] # DISCH_DISP\n",
    "data = pl.read_csv('EPIC_EMR/patient_information.csv',dtypes={})\n",
    "data = data.unique()\n",
    "data = data.with_columns(pl.col('DISCH_DISP_C').cast(pl.UInt64),pl.col('ASA_RATING_C').cast(pl.UInt64))\n",
    "\n",
    "two_columns_encoding = [('DISCH_DISP','DISCH_DISP_C'),('ASA_RATING','ASA_RATING_C')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col_1,col_2 in two_columns_encoding:\n",
    "    patient_information_encoding[col_1+'___'+col_2] = encode_two_columns(data,col_1,col_2)\n",
    "\n",
    "for col in cols_to_encode:\n",
    "    data, patient_information_encoding[col] = encode_column_and_replace(data,col)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 27)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>DISCH_DISP_C</th><th>DISCH_DISP</th><th>HOSP_ADMSN_TIME</th><th>HOSP_DISCH_TIME</th><th>LOS</th><th>ICU_ADMIN_FLAG</th><th>SURGERY_DATE</th><th>BIRTH_DATE</th><th>HEIGHT</th><th>WEIGHT</th><th>SEX</th><th>PRIMARY_ANES_TYPE_NM</th><th>ASA_RATING_C</th><th>ASA_RATING</th><th>PATIENT_CLASS_GROUP</th><th>PATIENT_CLASS_NM</th><th>PRIMARY_PROCEDURE_NM</th><th>IN_OR_DTTM</th><th>OUT_OR_DTTM</th><th>AN_START_DATETIME</th><th>AN_STOP_DATETIME</th><th>ICU_ADMIN_FLAG_encoded</th><th>SEX_encoded</th><th>PRIMARY_ANES_TYPE_NM_encoded</th><th>PATIENT_CLASS_GROUP_encoded</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>7</td><td>7</td><td>0</td><td>14</td><td>14</td><td>0</td><td>0</td><td>0</td><td>12726</td><td>2363</td><td>0</td><td>36</td><td>6811</td><td>6811</td><td>0</td><td>0</td><td>6</td><td>6403</td><td>6469</td><td>7313</td><td>7327</td><td>0</td><td>0</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 27)\n",
       "┌────────┬─────┬──────────────┬────────────┬───┬───────────────────┬─────────────┬──────────────────┬──────────────────┐\n",
       "│ LOG_ID ┆ MRN ┆ DISCH_DISP_C ┆ DISCH_DISP ┆ … ┆ ICU_ADMIN_FLAG_en ┆ SEX_encoded ┆ PRIMARY_ANES_TYP ┆ PATIENT_CLASS_GR │\n",
       "│ ---    ┆ --- ┆ ---          ┆ ---        ┆   ┆ coded             ┆ ---         ┆ E_NM_encoded     ┆ OUP_encoded      │\n",
       "│ u32    ┆ u32 ┆ u32          ┆ u32        ┆   ┆ ---               ┆ u32         ┆ ---              ┆ ---              │\n",
       "│        ┆     ┆              ┆            ┆   ┆ u32               ┆             ┆ u32              ┆ u32              │\n",
       "╞════════╪═════╪══════════════╪════════════╪═══╪═══════════════════╪═════════════╪══════════════════╪══════════════════╡\n",
       "│ 0      ┆ 0   ┆ 7            ┆ 7          ┆ … ┆ 0                 ┆ 0           ┆ 0                ┆ 0                │\n",
       "└────────┴─────┴──────────────┴────────────┴───┴───────────────────┴─────────────┴──────────────────┴──────────────────┘"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.null_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "data, patient_information_encoding['DISCH_DISP'] = encode_column_and_replace(data,'DISCH_DISP',patient_information_encoding['DISCH_DISP___DISCH_DISP_C'])\n",
    "data, patient_information_encoding['ASA_RATING'] = encode_column_and_replace(data,'ASA_RATING',patient_information_encoding['ASA_RATING___ASA_RATING_C'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 29)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>DISCH_DISP_C</th><th>DISCH_DISP</th><th>HOSP_ADMSN_TIME</th><th>HOSP_DISCH_TIME</th><th>LOS</th><th>ICU_ADMIN_FLAG</th><th>SURGERY_DATE</th><th>BIRTH_DATE</th><th>HEIGHT</th><th>WEIGHT</th><th>SEX</th><th>PRIMARY_ANES_TYPE_NM</th><th>ASA_RATING_C</th><th>ASA_RATING</th><th>PATIENT_CLASS_GROUP</th><th>PATIENT_CLASS_NM</th><th>PRIMARY_PROCEDURE_NM</th><th>IN_OR_DTTM</th><th>OUT_OR_DTTM</th><th>AN_START_DATETIME</th><th>AN_STOP_DATETIME</th><th>ICU_ADMIN_FLAG_encoded</th><th>SEX_encoded</th><th>PRIMARY_ANES_TYPE_NM_encoded</th><th>PATIENT_CLASS_GROUP_encoded</th><th>DISCH_DISP_encoded</th><th>ASA_RATING_encoded</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>7</td><td>7</td><td>0</td><td>14</td><td>14</td><td>0</td><td>0</td><td>0</td><td>12726</td><td>2363</td><td>0</td><td>36</td><td>6811</td><td>6811</td><td>0</td><td>0</td><td>6</td><td>6403</td><td>6469</td><td>7313</td><td>7327</td><td>0</td><td>0</td><td>0</td><td>0</td><td>7</td><td>6811</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 29)\n",
       "┌────────┬─────┬──────────────┬────────────┬───┬─────────────────┬─────────────────┬─────────────────┬─────────────────┐\n",
       "│ LOG_ID ┆ MRN ┆ DISCH_DISP_C ┆ DISCH_DISP ┆ … ┆ PRIMARY_ANES_TY ┆ PATIENT_CLASS_G ┆ DISCH_DISP_enco ┆ ASA_RATING_enco │\n",
       "│ ---    ┆ --- ┆ ---          ┆ ---        ┆   ┆ PE_NM_encoded   ┆ ROUP_encoded    ┆ ded             ┆ ded             │\n",
       "│ u32    ┆ u32 ┆ u32          ┆ u32        ┆   ┆ ---             ┆ ---             ┆ ---             ┆ ---             │\n",
       "│        ┆     ┆              ┆            ┆   ┆ u32             ┆ u32             ┆ u32             ┆ u32             │\n",
       "╞════════╪═════╪══════════════╪════════════╪═══╪═════════════════╪═════════════════╪═════════════════╪═════════════════╡\n",
       "│ 0      ┆ 0   ┆ 7            ┆ 7          ┆ … ┆ 0               ┆ 0               ┆ 7               ┆ 6811            │\n",
       "└────────┴─────┴──────────────┴────────────┴───┴─────────────────┴─────────────────┴─────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.null_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 29)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>DISCH_DISP_C</th><th>DISCH_DISP</th><th>HOSP_ADMSN_TIME</th><th>HOSP_DISCH_TIME</th><th>LOS</th><th>ICU_ADMIN_FLAG</th><th>SURGERY_DATE</th><th>BIRTH_DATE</th><th>HEIGHT</th><th>WEIGHT</th><th>SEX</th><th>PRIMARY_ANES_TYPE_NM</th><th>ASA_RATING_C</th><th>ASA_RATING</th><th>PATIENT_CLASS_GROUP</th><th>PATIENT_CLASS_NM</th><th>PRIMARY_PROCEDURE_NM</th><th>IN_OR_DTTM</th><th>OUT_OR_DTTM</th><th>AN_START_DATETIME</th><th>AN_STOP_DATETIME</th><th>ICU_ADMIN_FLAG_encoded</th><th>SEX_encoded</th><th>PRIMARY_ANES_TYPE_NM_encoded</th><th>PATIENT_CLASS_GROUP_encoded</th><th>DISCH_DISP_encoded</th><th>ASA_RATING_encoded</th></tr><tr><td>str</td><td>str</td><td>u64</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>u64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>u64</td><td>u64</td><td>u64</td><td>u64</td><td>u64</td><td>u64</td></tr></thead><tbody><tr><td>&quot;6af225abbb809c…</td><td>&quot;7c32ed97b90276…</td><td>20</td><td>&quot;Home Healthcar…</td><td>&quot;10/28/20 9:28&quot;</td><td>&quot;11/5/20 18:52&quot;</td><td>8.0</td><td>&quot;Yes&quot;</td><td>&quot;10/28/20 0:00&quot;</td><td>55</td><td>&quot;5&#x27; 0&quot;</td><td>2620.83</td><td>&quot;Female&quot;</td><td>&quot;Regional&quot;</td><td>3</td><td>&quot;Severe Systemi…</td><td>&quot;Inpatient&quot;</td><td>&quot;Hospital Inpat…</td><td>&quot;TRANSPLANT REC…</td><td>&quot;10/28/20 12:38…</td><td>&quot;10/28/20 16:02…</td><td>&quot;10/28/20 12:38…</td><td>&quot;10/28/20 16:08…</td><td>0</td><td>2</td><td>10</td><td>0</td><td>20</td><td>3</td></tr><tr><td>&quot;3119d72120dacb…</td><td>&quot;cdc958fff5570a…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;4/7/19 5:18&quot;</td><td>&quot;4/8/19 9:50&quot;</td><td>1.0</td><td>&quot;Yes&quot;</td><td>&quot;4/7/19 0:00&quot;</td><td>54</td><td>&quot;5&#x27; 2&quot;</td><td>2125.23</td><td>&quot;Female&quot;</td><td>&quot;General&quot;</td><td>2</td><td>&quot;Mild Systemic …</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;SACROCOLPOPEXY…</td><td>&quot;4/7/19 7:12&quot;</td><td>&quot;4/7/19 13:38&quot;</td><td>&quot;4/7/19 7:12&quot;</td><td>&quot;4/7/19 13:43&quot;</td><td>0</td><td>2</td><td>1</td><td>1</td><td>15</td><td>2</td></tr><tr><td>&quot;f0d16bb8e59321…</td><td>&quot;95d54ee9f5abfa…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;12/14/18 7:16&quot;</td><td>&quot;12/16/18 16:05…</td><td>2.0</td><td>&quot;No&quot;</td><td>&quot;12/14/18 0:00&quot;</td><td>43</td><td>&quot;5&#x27; 11&quot;</td><td>3710.78</td><td>&quot;Male&quot;</td><td>&quot;General&quot;</td><td>2</td><td>&quot;Mild Systemic …</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;URETHROPLASTY,…</td><td>&quot;12/14/18 8:44&quot;</td><td>&quot;12/14/18 12:14…</td><td>&quot;12/14/18 8:46&quot;</td><td>&quot;12/14/18 12:30…</td><td>1</td><td>1</td><td>1</td><td>1</td><td>15</td><td>2</td></tr><tr><td>&quot;32f1980e394417…</td><td>&quot;0dcace1cd2a4d8…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;6/22/19 8:10&quot;</td><td>&quot;6/23/19 11:15&quot;</td><td>1.0</td><td>&quot;No&quot;</td><td>&quot;6/22/19 0:00&quot;</td><td>79</td><td>null</td><td>1968.27</td><td>&quot;Male&quot;</td><td>&quot;General&quot;</td><td>3</td><td>&quot;Severe Systemi…</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;DECOMPRESSION,…</td><td>&quot;6/22/19 10:36&quot;</td><td>&quot;6/22/19 13:05&quot;</td><td>&quot;6/22/19 10:36&quot;</td><td>&quot;6/22/19 13:12&quot;</td><td>1</td><td>1</td><td>1</td><td>1</td><td>15</td><td>3</td></tr><tr><td>&quot;db83846371c937…</td><td>&quot;e3025ead155cb9…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;10/12/19 8:43&quot;</td><td>&quot;10/13/19 18:02…</td><td>1.0</td><td>&quot;No&quot;</td><td>&quot;10/12/19 0:00&quot;</td><td>67</td><td>&quot;5&#x27; 4.5&quot;</td><td>2567.92</td><td>&quot;Female&quot;</td><td>&quot;General&quot;</td><td>2</td><td>&quot;Mild Systemic …</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;HYSTERECTOMY, …</td><td>&quot;10/12/19 10:38…</td><td>&quot;10/12/19 14:31…</td><td>&quot;10/12/19 10:38…</td><td>&quot;10/12/19 14:37…</td><td>1</td><td>2</td><td>1</td><td>1</td><td>15</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 29)\n",
       "┌──────────────┬──────────────┬──────────────┬─────────────┬───┬─────────────┬─────────────┬─────────────┬─────────────┐\n",
       "│ LOG_ID       ┆ MRN          ┆ DISCH_DISP_C ┆ DISCH_DISP  ┆ … ┆ PRIMARY_ANE ┆ PATIENT_CLA ┆ DISCH_DISP_ ┆ ASA_RATING_ │\n",
       "│ ---          ┆ ---          ┆ ---          ┆ ---         ┆   ┆ S_TYPE_NM_e ┆ SS_GROUP_en ┆ encoded     ┆ encoded     │\n",
       "│ str          ┆ str          ┆ u64          ┆ str         ┆   ┆ ncoded      ┆ coded       ┆ ---         ┆ ---         │\n",
       "│              ┆              ┆              ┆             ┆   ┆ ---         ┆ ---         ┆ u64         ┆ u64         │\n",
       "│              ┆              ┆              ┆             ┆   ┆ u64         ┆ u64         ┆             ┆             │\n",
       "╞══════════════╪══════════════╪══════════════╪═════════════╪═══╪═════════════╪═════════════╪═════════════╪═════════════╡\n",
       "│ 6af225abbb80 ┆ 7c32ed97b902 ┆ 20           ┆ Home        ┆ … ┆ 10          ┆ 0           ┆ 20          ┆ 3           │\n",
       "│ 9cec         ┆ 76e9         ┆              ┆ Healthcare  ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆              ┆ IP Admit    ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆              ┆ Related     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ 3119d72120da ┆ cdc958fff557 ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 1           ┆ 15          ┆ 2           │\n",
       "│ cb7c         ┆ 0ab0         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ f0d16bb8e593 ┆ 95d54ee9f5ab ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 1           ┆ 15          ┆ 2           │\n",
       "│ 218e         ┆ fab3         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ 32f1980e3944 ┆ 0dcace1cd2a4 ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 1           ┆ 15          ┆ 3           │\n",
       "│ 170e         ┆ d89d         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ db83846371c9 ┆ e3025ead155c ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 1           ┆ 15          ┆ 2           │\n",
       "│ 37a9         ┆ b9bb         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "└──────────────┴──────────────┴──────────────┴─────────────┴───┴─────────────┴─────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Height is fiven in feet'inch metric. We convert this into centimeters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.with_columns(pl.col('HEIGHT').str.split(by=\"'\").map_elements(lambda x: float(x[0])*30.48  + float(x[1])*2.54).alias('HEIGHT_IN_CM'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 30)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>DISCH_DISP_C</th><th>DISCH_DISP</th><th>HOSP_ADMSN_TIME</th><th>HOSP_DISCH_TIME</th><th>LOS</th><th>ICU_ADMIN_FLAG</th><th>SURGERY_DATE</th><th>BIRTH_DATE</th><th>HEIGHT</th><th>WEIGHT</th><th>SEX</th><th>PRIMARY_ANES_TYPE_NM</th><th>ASA_RATING_C</th><th>ASA_RATING</th><th>PATIENT_CLASS_GROUP</th><th>PATIENT_CLASS_NM</th><th>PRIMARY_PROCEDURE_NM</th><th>IN_OR_DTTM</th><th>OUT_OR_DTTM</th><th>AN_START_DATETIME</th><th>AN_STOP_DATETIME</th><th>ICU_ADMIN_FLAG_encoded</th><th>SEX_encoded</th><th>PRIMARY_ANES_TYPE_NM_encoded</th><th>PATIENT_CLASS_GROUP_encoded</th><th>DISCH_DISP_encoded</th><th>ASA_RATING_encoded</th><th>HEIGHT_IN_CM</th></tr><tr><td>str</td><td>str</td><td>u64</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>u64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>u64</td><td>u64</td><td>u64</td><td>u64</td><td>u64</td><td>u64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;6af225abbb809c…</td><td>&quot;7c32ed97b90276…</td><td>20</td><td>&quot;Home Healthcar…</td><td>&quot;10/28/20 9:28&quot;</td><td>&quot;11/5/20 18:52&quot;</td><td>8.0</td><td>&quot;Yes&quot;</td><td>&quot;10/28/20 0:00&quot;</td><td>55</td><td>&quot;5&#x27; 0&quot;</td><td>2620.83</td><td>&quot;Female&quot;</td><td>&quot;Regional&quot;</td><td>3</td><td>&quot;Severe Systemi…</td><td>&quot;Inpatient&quot;</td><td>&quot;Hospital Inpat…</td><td>&quot;TRANSPLANT REC…</td><td>&quot;10/28/20 12:38…</td><td>&quot;10/28/20 16:02…</td><td>&quot;10/28/20 12:38…</td><td>&quot;10/28/20 16:08…</td><td>0</td><td>2</td><td>10</td><td>0</td><td>20</td><td>3</td><td>152.4</td></tr><tr><td>&quot;3119d72120dacb…</td><td>&quot;cdc958fff5570a…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;4/7/19 5:18&quot;</td><td>&quot;4/8/19 9:50&quot;</td><td>1.0</td><td>&quot;Yes&quot;</td><td>&quot;4/7/19 0:00&quot;</td><td>54</td><td>&quot;5&#x27; 2&quot;</td><td>2125.23</td><td>&quot;Female&quot;</td><td>&quot;General&quot;</td><td>2</td><td>&quot;Mild Systemic …</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;SACROCOLPOPEXY…</td><td>&quot;4/7/19 7:12&quot;</td><td>&quot;4/7/19 13:38&quot;</td><td>&quot;4/7/19 7:12&quot;</td><td>&quot;4/7/19 13:43&quot;</td><td>0</td><td>2</td><td>1</td><td>1</td><td>15</td><td>2</td><td>157.48</td></tr><tr><td>&quot;f0d16bb8e59321…</td><td>&quot;95d54ee9f5abfa…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;12/14/18 7:16&quot;</td><td>&quot;12/16/18 16:05…</td><td>2.0</td><td>&quot;No&quot;</td><td>&quot;12/14/18 0:00&quot;</td><td>43</td><td>&quot;5&#x27; 11&quot;</td><td>3710.78</td><td>&quot;Male&quot;</td><td>&quot;General&quot;</td><td>2</td><td>&quot;Mild Systemic …</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;URETHROPLASTY,…</td><td>&quot;12/14/18 8:44&quot;</td><td>&quot;12/14/18 12:14…</td><td>&quot;12/14/18 8:46&quot;</td><td>&quot;12/14/18 12:30…</td><td>1</td><td>1</td><td>1</td><td>1</td><td>15</td><td>2</td><td>180.34</td></tr><tr><td>&quot;32f1980e394417…</td><td>&quot;0dcace1cd2a4d8…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;6/22/19 8:10&quot;</td><td>&quot;6/23/19 11:15&quot;</td><td>1.0</td><td>&quot;No&quot;</td><td>&quot;6/22/19 0:00&quot;</td><td>79</td><td>null</td><td>1968.27</td><td>&quot;Male&quot;</td><td>&quot;General&quot;</td><td>3</td><td>&quot;Severe Systemi…</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;DECOMPRESSION,…</td><td>&quot;6/22/19 10:36&quot;</td><td>&quot;6/22/19 13:05&quot;</td><td>&quot;6/22/19 10:36&quot;</td><td>&quot;6/22/19 13:12&quot;</td><td>1</td><td>1</td><td>1</td><td>1</td><td>15</td><td>3</td><td>null</td></tr><tr><td>&quot;db83846371c937…</td><td>&quot;e3025ead155cb9…</td><td>15</td><td>&quot;Home Routine&quot;</td><td>&quot;10/12/19 8:43&quot;</td><td>&quot;10/13/19 18:02…</td><td>1.0</td><td>&quot;No&quot;</td><td>&quot;10/12/19 0:00&quot;</td><td>67</td><td>&quot;5&#x27; 4.5&quot;</td><td>2567.92</td><td>&quot;Female&quot;</td><td>&quot;General&quot;</td><td>2</td><td>&quot;Mild Systemic …</td><td>&quot;Outpatient&quot;</td><td>&quot;Hospital Outpa…</td><td>&quot;HYSTERECTOMY, …</td><td>&quot;10/12/19 10:38…</td><td>&quot;10/12/19 14:31…</td><td>&quot;10/12/19 10:38…</td><td>&quot;10/12/19 14:37…</td><td>1</td><td>2</td><td>1</td><td>1</td><td>15</td><td>2</td><td>163.83</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 30)\n",
       "┌──────────────┬──────────────┬──────────────┬─────────────┬───┬─────────────┬─────────────┬─────────────┬─────────────┐\n",
       "│ LOG_ID       ┆ MRN          ┆ DISCH_DISP_C ┆ DISCH_DISP  ┆ … ┆ PATIENT_CLA ┆ DISCH_DISP_ ┆ ASA_RATING_ ┆ HEIGHT_IN_C │\n",
       "│ ---          ┆ ---          ┆ ---          ┆ ---         ┆   ┆ SS_GROUP_en ┆ encoded     ┆ encoded     ┆ M           │\n",
       "│ str          ┆ str          ┆ u64          ┆ str         ┆   ┆ coded       ┆ ---         ┆ ---         ┆ ---         │\n",
       "│              ┆              ┆              ┆             ┆   ┆ ---         ┆ u64         ┆ u64         ┆ f64         │\n",
       "│              ┆              ┆              ┆             ┆   ┆ u64         ┆             ┆             ┆             │\n",
       "╞══════════════╪══════════════╪══════════════╪═════════════╪═══╪═════════════╪═════════════╪═════════════╪═════════════╡\n",
       "│ 6af225abbb80 ┆ 7c32ed97b902 ┆ 20           ┆ Home        ┆ … ┆ 0           ┆ 20          ┆ 3           ┆ 152.4       │\n",
       "│ 9cec         ┆ 76e9         ┆              ┆ Healthcare  ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆              ┆ IP Admit    ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆              ┆ Related     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ 3119d72120da ┆ cdc958fff557 ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 15          ┆ 2           ┆ 157.48      │\n",
       "│ cb7c         ┆ 0ab0         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ f0d16bb8e593 ┆ 95d54ee9f5ab ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 15          ┆ 2           ┆ 180.34      │\n",
       "│ 218e         ┆ fab3         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ 32f1980e3944 ┆ 0dcace1cd2a4 ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 15          ┆ 3           ┆ null        │\n",
       "│ 170e         ┆ d89d         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "│ db83846371c9 ┆ e3025ead155c ┆ 15           ┆ Home        ┆ … ┆ 1           ┆ 15          ┆ 2           ┆ 163.83      │\n",
       "│ 37a9         ┆ b9bb         ┆              ┆ Routine     ┆   ┆             ┆             ┆             ┆             │\n",
       "└──────────────┴──────────────┴──────────────┴─────────────┴───┴─────────────┴─────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We filter the columns that provide useful information for our analysis and models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_needed=[\n",
    "    'LOG_ID',\n",
    "    'MRN',\n",
    " 'HOSP_ADMSN_TIME',\n",
    " 'HOSP_DISCH_TIME',\n",
    " 'LOS',\n",
    " 'SURGERY_DATE',\n",
    " 'BIRTH_DATE',\n",
    " 'WEIGHT',\n",
    " 'IN_OR_DTTM',\n",
    " 'OUT_OR_DTTM',\n",
    " 'AN_START_DATETIME',\n",
    " 'AN_STOP_DATETIME',\n",
    " 'ICU_ADMIN_FLAG_encoded',\n",
    " 'SEX_encoded',\n",
    " 'PRIMARY_ANES_TYPE_NM_encoded',\n",
    " 'PATIENT_CLASS_GROUP_encoded',\n",
    " 'DISCH_DISP_encoded',\n",
    " 'ASA_RATING_encoded',\n",
    " 'HEIGHT_IN_CM']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(columns_needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64364, 19)\n"
     ]
    }
   ],
   "source": [
    "data = data.select(columns_needed)\n",
    "print(data.shape)\n",
    "data.write_csv('EPIC_EMR_cleaned/patient_information_cleaned.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning patient lab data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a very large csv file with the size of 3GB. This file contains lab samples of every patient. The column 'Abnormal Flag' is created by lab faculty based on all the other attributes of the patient, which means it can summarise all the other attributes. Thus, we can omit rest of the columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_lab_encoding = {}\n",
    "cols_to_encode = ['Abnormal Flag'] # DISCH_DISP\n",
    "data = pl.read_csv('EPIC_EMR/patient_labs.csv',dtypes={})\n",
    "data = data.unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 10)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>ENC_TYPE_NM</th><th>Lab Code</th><th>Lab Name</th><th>Observation Value</th><th>Measurement Units</th><th>Reference Range</th><th>Abnormal Flag</th><th>Collection Datetime</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;2028-9&quot;</td><td>&quot;Carbon dioxide…</td><td>24.0</td><td>&quot;mmol/L&quot;</td><td>&quot;21-31&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-10 23:…</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;2951-2&quot;</td><td>&quot;Sodium&quot;</td><td>136.0</td><td>&quot;mmol/L&quot;</td><td>&quot;136-145&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-08 04:…</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;30428-7&quot;</td><td>&quot;Erythrocyte me…</td><td>92.1</td><td>&quot;FL&quot;</td><td>&quot;81.5-97.0&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-09 05:…</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;48642-3&quot;</td><td>&quot;Glomerular fil…</td><td>9.999999e6</td><td>&quot;Unknown&quot;</td><td>&quot;Unknown&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-12 21:…</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;48643-1&quot;</td><td>&quot;Glomerular fil…</td><td>9.999999e6</td><td>&quot;Unknown&quot;</td><td>&quot;Unknown&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-12 21:…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 10)\n",
       "┌───────────────────┬───────────────────┬─────────────┬──────────┬───┬─────────────┬───────────┬──────────┬────────────┐\n",
       "│ LOG_ID            ┆ MRN               ┆ ENC_TYPE_NM ┆ Lab Code ┆ … ┆ Measurement ┆ Reference ┆ Abnormal ┆ Collection │\n",
       "│ ---               ┆ ---               ┆ ---         ┆ ---      ┆   ┆ Units       ┆ Range     ┆ Flag     ┆ Datetime   │\n",
       "│ str               ┆ str               ┆ str         ┆ str      ┆   ┆ ---         ┆ ---       ┆ ---      ┆ ---        │\n",
       "│                   ┆                   ┆             ┆          ┆   ┆ str         ┆ str       ┆ str      ┆ str        │\n",
       "╞═══════════════════╪═══════════════════╪═════════════╪══════════╪═══╪═════════════╪═══════════╪══════════╪════════════╡\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee  ┆ Hospital    ┆ 2028-9   ┆ … ┆ mmol/L      ┆ 21-31     ┆ N        ┆ 2020-03-10 │\n",
       "│                   ┆                   ┆ Encounter   ┆          ┆   ┆             ┆           ┆          ┆ 23:36:00   │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee  ┆ Hospital    ┆ 2951-2   ┆ … ┆ mmol/L      ┆ 136-145   ┆ N        ┆ 2020-03-08 │\n",
       "│                   ┆                   ┆ Encounter   ┆          ┆   ┆             ┆           ┆          ┆ 04:09:00   │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee  ┆ Hospital    ┆ 30428-7  ┆ … ┆ FL          ┆ 81.5-97.0 ┆ N        ┆ 2020-03-09 │\n",
       "│                   ┆                   ┆ Encounter   ┆          ┆   ┆             ┆           ┆          ┆ 05:33:00   │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee  ┆ Hospital    ┆ 48642-3  ┆ … ┆ Unknown     ┆ Unknown   ┆ N        ┆ 2020-03-12 │\n",
       "│                   ┆                   ┆ Encounter   ┆          ┆   ┆             ┆           ┆          ┆ 21:36:00   │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee  ┆ Hospital    ┆ 48643-1  ┆ … ┆ Unknown     ┆ Unknown   ┆ N        ┆ 2020-03-12 │\n",
       "│                   ┆                   ┆ Encounter   ┆          ┆   ┆             ┆           ┆          ┆ 21:36:00   │\n",
       "└───────────────────┴───────────────────┴─────────────┴──────────┴───┴─────────────┴───────────┴──────────┴────────────┘"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in cols_to_encode:\n",
    "    data, patient_lab_encoding[col] = encode_column_and_replace(data,col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 11)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>ENC_TYPE_NM</th><th>Lab Code</th><th>Lab Name</th><th>Observation Value</th><th>Measurement Units</th><th>Reference Range</th><th>Abnormal Flag</th><th>Collection Datetime</th><th>Abnormal Flag_encoded</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>89706</td><td>0</td><td>0</td><td>201566</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 11)\n",
       "┌────────┬─────┬─────────────┬──────────┬───┬─────────────────┬───────────────┬────────────┬───────────────────────┐\n",
       "│ LOG_ID ┆ MRN ┆ ENC_TYPE_NM ┆ Lab Code ┆ … ┆ Reference Range ┆ Abnormal Flag ┆ Collection ┆ Abnormal Flag_encoded │\n",
       "│ ---    ┆ --- ┆ ---         ┆ ---      ┆   ┆ ---             ┆ ---           ┆ Datetime   ┆ ---                   │\n",
       "│ u32    ┆ u32 ┆ u32         ┆ u32      ┆   ┆ u32             ┆ u32           ┆ ---        ┆ u32                   │\n",
       "│        ┆     ┆             ┆          ┆   ┆                 ┆               ┆ u32        ┆                       │\n",
       "╞════════╪═════╪═════════════╪══════════╪═══╪═════════════════╪═══════════════╪════════════╪═══════════════════════╡\n",
       "│ 0      ┆ 0   ┆ 0           ┆ 0        ┆ … ┆ 0               ┆ 201566        ┆ 0          ┆ 0                     │\n",
       "└────────┴─────┴─────────────┴──────────┴───┴─────────────────┴───────────────┴────────────┴───────────────────────┘"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.null_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 11)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>ENC_TYPE_NM</th><th>Lab Code</th><th>Lab Name</th><th>Observation Value</th><th>Measurement Units</th><th>Reference Range</th><th>Abnormal Flag</th><th>Collection Datetime</th><th>Abnormal Flag_encoded</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>u64</td></tr></thead><tbody><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;2028-9&quot;</td><td>&quot;Carbon dioxide…</td><td>24.0</td><td>&quot;mmol/L&quot;</td><td>&quot;21-31&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-10 23:…</td><td>1</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;2951-2&quot;</td><td>&quot;Sodium&quot;</td><td>136.0</td><td>&quot;mmol/L&quot;</td><td>&quot;136-145&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-08 04:…</td><td>1</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;30428-7&quot;</td><td>&quot;Erythrocyte me…</td><td>92.1</td><td>&quot;FL&quot;</td><td>&quot;81.5-97.0&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-09 05:…</td><td>1</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;48642-3&quot;</td><td>&quot;Glomerular fil…</td><td>9.999999e6</td><td>&quot;Unknown&quot;</td><td>&quot;Unknown&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-12 21:…</td><td>1</td></tr><tr><td>&quot;3ab8e6344cc9f9…</td><td>&quot;559c869f9d7db8…</td><td>&quot;Hospital Encou…</td><td>&quot;48643-1&quot;</td><td>&quot;Glomerular fil…</td><td>9.999999e6</td><td>&quot;Unknown&quot;</td><td>&quot;Unknown&quot;</td><td>&quot;N&quot;</td><td>&quot;2020-03-12 21:…</td><td>1</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 11)\n",
       "┌───────────────────┬──────────────────┬─────────────┬──────────┬───┬───────────┬──────────┬────────────┬──────────────┐\n",
       "│ LOG_ID            ┆ MRN              ┆ ENC_TYPE_NM ┆ Lab Code ┆ … ┆ Reference ┆ Abnormal ┆ Collection ┆ Abnormal     │\n",
       "│ ---               ┆ ---              ┆ ---         ┆ ---      ┆   ┆ Range     ┆ Flag     ┆ Datetime   ┆ Flag_encoded │\n",
       "│ str               ┆ str              ┆ str         ┆ str      ┆   ┆ ---       ┆ ---      ┆ ---        ┆ ---          │\n",
       "│                   ┆                  ┆             ┆          ┆   ┆ str       ┆ str      ┆ str        ┆ u64          │\n",
       "╞═══════════════════╪══════════════════╪═════════════╪══════════╪═══╪═══════════╪══════════╪════════════╪══════════════╡\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee ┆ Hospital    ┆ 2028-9   ┆ … ┆ 21-31     ┆ N        ┆ 2020-03-10 ┆ 1            │\n",
       "│                   ┆                  ┆ Encounter   ┆          ┆   ┆           ┆          ┆ 23:36:00   ┆              │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee ┆ Hospital    ┆ 2951-2   ┆ … ┆ 136-145   ┆ N        ┆ 2020-03-08 ┆ 1            │\n",
       "│                   ┆                  ┆ Encounter   ┆          ┆   ┆           ┆          ┆ 04:09:00   ┆              │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee ┆ Hospital    ┆ 30428-7  ┆ … ┆ 81.5-97.0 ┆ N        ┆ 2020-03-09 ┆ 1            │\n",
       "│                   ┆                  ┆ Encounter   ┆          ┆   ┆           ┆          ┆ 05:33:00   ┆              │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee ┆ Hospital    ┆ 48642-3  ┆ … ┆ Unknown   ┆ N        ┆ 2020-03-12 ┆ 1            │\n",
       "│                   ┆                  ┆ Encounter   ┆          ┆   ┆           ┆          ┆ 21:36:00   ┆              │\n",
       "│ 3ab8e6344cc9f954  ┆ 559c869f9d7db8ee ┆ Hospital    ┆ 48643-1  ┆ … ┆ Unknown   ┆ N        ┆ 2020-03-12 ┆ 1            │\n",
       "│                   ┆                  ┆ Encounter   ┆          ┆   ┆           ┆          ┆ 21:36:00   ┆              │\n",
       "└───────────────────┴──────────────────┴─────────────┴──────────┴───┴───────────┴──────────┴────────────┴──────────────┘"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(29071808, 4)\n"
     ]
    }
   ],
   "source": [
    "columns_needed = ['Abnormal Flag_encoded','Collection Datetime','LOG_ID','MRN']\n",
    "data = data.select(columns_needed)\n",
    "print(data.shape)\n",
    "data.write_csv('EPIC_EMR_cleaned/patient_labs_cleaned.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning patient post op complications"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This Dataset contains the post operative complications observed by the patients. All of the columns provide useful information for our analysis, which must be encoded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_post_op_encoding = {}\n",
    "cols_to_encode = ['CONTEXT_NAME','Element_abbr','SMRTDTA_ELEM_VALUE'] \n",
    "data = pl.read_csv('EPIC_EMR/patient_post_op_complications.csv',dtypes={})\n",
    "data = data.unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>Element_Name</th><th>CONTEXT_NAME</th><th>Element_abbr</th><th>SMRTDTA_ELEM_VALUE</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;b1df0a3b9037bd…</td><td>&quot;1a479b8bad2165…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;AN Post-op Com…</td><td>&quot;None&quot;</td></tr><tr><td>&quot;567f27d6b31690…</td><td>&quot;f35086637e7870…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;AN Post-op Com…</td><td>&quot;None&quot;</td></tr><tr><td>&quot;0059e815691350…</td><td>&quot;73acc50cd25826…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;NOTE&quot;</td><td>&quot;AN Post-op Com…</td><td>&quot;None&quot;</td></tr><tr><td>&quot;a7a15da2257283…</td><td>&quot;aee6ffad9c9ecc…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;AN Post-op Com…</td><td>&quot;None&quot;</td></tr><tr><td>&quot;00072a39c8cfa0…</td><td>&quot;c88dcd13cb0cf4…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;NOTE&quot;</td><td>&quot;AN Post-op Com…</td><td>&quot;None&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌──────────────────┬──────────────────┬────────────────┬──────────────┬───────────────────────────┬────────────────────┐\n",
       "│ LOG_ID           ┆ MRN              ┆ Element_Name   ┆ CONTEXT_NAME ┆ Element_abbr              ┆ SMRTDTA_ELEM_VALUE │\n",
       "│ ---              ┆ ---              ┆ ---            ┆ ---          ┆ ---                       ┆ ---                │\n",
       "│ str              ┆ str              ┆ str            ┆ str          ┆ str                       ┆ str                │\n",
       "╞══════════════════╪══════════════════╪════════════════╪══════════════╪═══════════════════════════╪════════════════════╡\n",
       "│ b1df0a3b9037bd60 ┆ 1a479b8bad216511 ┆ AN AQI POST-OP ┆ ENCOUNTER    ┆ AN Post-op Complications  ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆                           ┆                    │\n",
       "│ 567f27d6b3169025 ┆ f35086637e787086 ┆ AN AQI POST-OP ┆ ENCOUNTER    ┆ AN Post-op Complications  ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆                           ┆                    │\n",
       "│ 0059e8156913502f ┆ 73acc50cd2582661 ┆ AN AQI POST-OP ┆ NOTE         ┆ AN Post-op Complications  ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆                           ┆                    │\n",
       "│ a7a15da22572839d ┆ aee6ffad9c9ecc9e ┆ AN AQI POST-OP ┆ ENCOUNTER    ┆ AN Post-op Complications  ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆                           ┆                    │\n",
       "│ 00072a39c8cfa0b6 ┆ c88dcd13cb0cf464 ┆ AN AQI POST-OP ┆ NOTE         ┆ AN Post-op Complications  ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆                           ┆                    │\n",
       "└──────────────────┴──────────────────┴────────────────┴──────────────┴───────────────────────────┴────────────────────┘"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>Element_Name</th><th>CONTEXT_NAME</th><th>Element_abbr</th><th>SMRTDTA_ELEM_VALUE</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;b1df0a3b9037bd…</td><td>&quot;1a479b8bad2165…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td></tr><tr><td>&quot;567f27d6b31690…</td><td>&quot;f35086637e7870…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td></tr><tr><td>&quot;0059e815691350…</td><td>&quot;73acc50cd25826…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;NOTE&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td></tr><tr><td>&quot;a7a15da2257283…</td><td>&quot;aee6ffad9c9ecc…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td></tr><tr><td>&quot;00072a39c8cfa0…</td><td>&quot;c88dcd13cb0cf4…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;NOTE&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌──────────────────┬──────────────────┬────────────────┬──────────────┬──────────────┬────────────────────┐\n",
       "│ LOG_ID           ┆ MRN              ┆ Element_Name   ┆ CONTEXT_NAME ┆ Element_abbr ┆ SMRTDTA_ELEM_VALUE │\n",
       "│ ---              ┆ ---              ┆ ---            ┆ ---          ┆ ---          ┆ ---                │\n",
       "│ str              ┆ str              ┆ str            ┆ str          ┆ str          ┆ str                │\n",
       "╞══════════════════╪══════════════════╪════════════════╪══════════════╪══════════════╪════════════════════╡\n",
       "│ b1df0a3b9037bd60 ┆ 1a479b8bad216511 ┆ AN AQI POST-OP ┆ ENCOUNTER    ┆              ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆              ┆                    │\n",
       "│ 567f27d6b3169025 ┆ f35086637e787086 ┆ AN AQI POST-OP ┆ ENCOUNTER    ┆              ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆              ┆                    │\n",
       "│ 0059e8156913502f ┆ 73acc50cd2582661 ┆ AN AQI POST-OP ┆ NOTE         ┆              ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆              ┆                    │\n",
       "│ a7a15da22572839d ┆ aee6ffad9c9ecc9e ┆ AN AQI POST-OP ┆ ENCOUNTER    ┆              ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆              ┆                    │\n",
       "│ 00072a39c8cfa0b6 ┆ c88dcd13cb0cf464 ┆ AN AQI POST-OP ┆ NOTE         ┆              ┆ None               │\n",
       "│                  ┆                  ┆ COMPLICATIONS  ┆              ┆              ┆                    │\n",
       "└──────────────────┴──────────────────┴────────────────┴──────────────┴──────────────┴────────────────────┘"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.with_columns(pl.col('Element_abbr').str.replace(r\"AN Post-op Complications\",''))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in cols_to_encode:\n",
    "    data, patient_post_op_encoding[col] = encode_column_and_replace(data,col)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>Element_Name</th><th>CONTEXT_NAME</th><th>Element_abbr</th><th>SMRTDTA_ELEM_VALUE</th><th>CONTEXT_NAME_encoded</th><th>Element_abbr_encoded</th><th>SMRTDTA_ELEM_VALUE_encoded</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>u64</td><td>u64</td><td>u64</td></tr></thead><tbody><tr><td>&quot;a8b3f9afa63c96…</td><td>&quot;aaebaad9587efc…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td><td>2</td><td>2</td><td>63</td></tr><tr><td>&quot;5dc0d50fa1d11e…</td><td>&quot;dc6e2920190557…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td><td>2</td><td>2</td><td>63</td></tr><tr><td>&quot;c66c0c773c2280…</td><td>&quot;66d9279ff5f4b9…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td><td>2</td><td>2</td><td>63</td></tr><tr><td>&quot;bec65b5c7ae241…</td><td>&quot;c80dbde76c38ec…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;Administrative…</td><td>2</td><td>2</td><td>233</td></tr><tr><td>&quot;104d64f326946e…</td><td>&quot;bb42a6512d570b…</td><td>&quot;AN AQI POST-OP…</td><td>&quot;ENCOUNTER&quot;</td><td>&quot;&quot;</td><td>&quot;None&quot;</td><td>2</td><td>2</td><td>63</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 9)\n",
       "┌──────────────┬──────────────┬──────────────┬─────────────┬───┬─────────────┬─────────────┬─────────────┬─────────────┐\n",
       "│ LOG_ID       ┆ MRN          ┆ Element_Name ┆ CONTEXT_NAM ┆ … ┆ SMRTDTA_ELE ┆ CONTEXT_NAM ┆ Element_abb ┆ SMRTDTA_ELE │\n",
       "│ ---          ┆ ---          ┆ ---          ┆ E           ┆   ┆ M_VALUE     ┆ E_encoded   ┆ r_encoded   ┆ M_VALUE_enc │\n",
       "│ str          ┆ str          ┆ str          ┆ ---         ┆   ┆ ---         ┆ ---         ┆ ---         ┆ oded        │\n",
       "│              ┆              ┆              ┆ str         ┆   ┆ str         ┆ u64         ┆ u64         ┆ ---         │\n",
       "│              ┆              ┆              ┆             ┆   ┆             ┆             ┆             ┆ u64         │\n",
       "╞══════════════╪══════════════╪══════════════╪═════════════╪═══╪═════════════╪═════════════╪═════════════╪═════════════╡\n",
       "│ a8b3f9afa63c ┆ aaebaad9587e ┆ AN AQI       ┆ ENCOUNTER   ┆ … ┆ None        ┆ 2           ┆ 2           ┆ 63          │\n",
       "│ 96d1         ┆ fc99         ┆ POST-OP COMP ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆ LICATIONS    ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│ 5dc0d50fa1d1 ┆ dc6e29201905 ┆ AN AQI       ┆ ENCOUNTER   ┆ … ┆ None        ┆ 2           ┆ 2           ┆ 63          │\n",
       "│ 1ea2         ┆ 577b         ┆ POST-OP COMP ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆ LICATIONS    ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│ c66c0c773c22 ┆ 66d9279ff5f4 ┆ AN AQI       ┆ ENCOUNTER   ┆ … ┆ None        ┆ 2           ┆ 2           ┆ 63          │\n",
       "│ 80d8         ┆ b927         ┆ POST-OP COMP ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆ LICATIONS    ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│ bec65b5c7ae2 ┆ c80dbde76c38 ┆ AN AQI       ┆ ENCOUNTER   ┆ … ┆ Administrat ┆ 2           ┆ 2           ┆ 233         │\n",
       "│ 41b8         ┆ ecad         ┆ POST-OP COMP ┆             ┆   ┆ ive         ┆             ┆             ┆             │\n",
       "│              ┆              ┆ LICATIONS    ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│ 104d64f32694 ┆ bb42a6512d57 ┆ AN AQI       ┆ ENCOUNTER   ┆ … ┆ None        ┆ 2           ┆ 2           ┆ 63          │\n",
       "│ 6e24         ┆ 0bbc         ┆ POST-OP COMP ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "│              ┆              ┆ LICATIONS    ┆             ┆   ┆             ┆             ┆             ┆             │\n",
       "└──────────────┴──────────────┴──────────────┴─────────────┴───┴─────────────┴─────────────┴─────────────┴─────────────┘"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>Element_Name</th><th>CONTEXT_NAME</th><th>Element_abbr</th><th>SMRTDTA_ELEM_VALUE</th><th>CONTEXT_NAME_encoded</th><th>Element_abbr_encoded</th><th>SMRTDTA_ELEM_VALUE_encoded</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>483</td><td>0</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 9)\n",
       "┌────────┬─────┬──────────────┬──────────────┬───┬─────────────────┬─────────────────┬────────────────┬────────────────┐\n",
       "│ LOG_ID ┆ MRN ┆ Element_Name ┆ CONTEXT_NAME ┆ … ┆ SMRTDTA_ELEM_VA ┆ CONTEXT_NAME_en ┆ Element_abbr_e ┆ SMRTDTA_ELEM_V │\n",
       "│ ---    ┆ --- ┆ ---          ┆ ---          ┆   ┆ LUE             ┆ coded           ┆ ncoded         ┆ ALUE_encoded   │\n",
       "│ u32    ┆ u32 ┆ u32          ┆ u32          ┆   ┆ ---             ┆ ---             ┆ ---            ┆ ---            │\n",
       "│        ┆     ┆              ┆              ┆   ┆ u32             ┆ u32             ┆ u32            ┆ u32            │\n",
       "╞════════╪═════╪══════════════╪══════════════╪═══╪═════════════════╪═════════════════╪════════════════╪════════════════╡\n",
       "│ 0      ┆ 0   ┆ 0            ┆ 0            ┆ … ┆ 483             ┆ 0               ┆ 0              ┆ 0              │\n",
       "└────────┴─────┴──────────────┴──────────────┴───┴─────────────────┴─────────────────┴────────────────┴────────────────┘"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.null_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(84950, 5)\n"
     ]
    }
   ],
   "source": [
    "columns_needed = ['SMRTDTA_ELEM_VALUE_encoded','Element_abbr_encoded','CONTEXT_NAME_encoded','LOG_ID','MRN']\n",
    "data = data.select(columns_needed)\n",
    "print(data.shape)\n",
    "data.write_csv('EPIC_EMR_cleaned/patient_post_op_complication_cleaned.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning Patient Vist\n",
    "This dataset contains the diagnosis information of all visits of patients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_visit_encoding = {}\n",
    "cols_to_encode = ['dx_name'] \n",
    "data = pl.read_csv('EPIC_EMR/patient_visit.csv',dtypes={'diagnosis_code':pl.Utf8})\n",
    "data = data.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>mrn</th><th>diagnosis_code</th><th>dx_name</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;fa149b1b36855b…</td><td>&quot;ad8291a53a44f0…</td><td>&quot;153.3&quot;</td><td>&quot;Malignant neop…</td></tr><tr><td>&quot;5e2775d183fc11…</td><td>&quot;5f3940d268bee2…</td><td>&quot;153.3&quot;</td><td>&quot;Malignant neop…</td></tr><tr><td>&quot;1cb372141215f6…</td><td>&quot;3668823554b94e…</td><td>&quot;188.0&quot;</td><td>&quot;Malignant neop…</td></tr><tr><td>&quot;f091dc2a1a7965…</td><td>&quot;9736aa7fdd136c…</td><td>&quot;188.0&quot;</td><td>&quot;Malignant neop…</td></tr><tr><td>&quot;0c357a71be7d7d…</td><td>&quot;c09ed71d87105c…</td><td>&quot;188.0&quot;</td><td>&quot;Malignant neop…</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌──────────────────┬──────────────────┬────────────────┬───────────────────────────────────┐\n",
       "│ LOG_ID           ┆ mrn              ┆ diagnosis_code ┆ dx_name                           │\n",
       "│ ---              ┆ ---              ┆ ---            ┆ ---                               │\n",
       "│ str              ┆ str              ┆ str            ┆ str                               │\n",
       "╞══════════════════╪══════════════════╪════════════════╪═══════════════════════════════════╡\n",
       "│ fa149b1b36855b20 ┆ ad8291a53a44f099 ┆ 153.3          ┆ Malignant neoplasm of sigmoid co… │\n",
       "│ 5e2775d183fc11d3 ┆ 5f3940d268bee23e ┆ 153.3          ┆ Malignant neoplasm of sigmoid co… │\n",
       "│ 1cb372141215f6e2 ┆ 3668823554b94ea1 ┆ 188.0          ┆ Malignant neoplasm of trigone of… │\n",
       "│ f091dc2a1a796519 ┆ 9736aa7fdd136c53 ┆ 188.0          ┆ Malignant neoplasm of trigone of… │\n",
       "│ 0c357a71be7d7d13 ┆ c09ed71d87105ca5 ┆ 188.0          ┆ Malignant neoplasm of trigone of… │\n",
       "└──────────────────┴──────────────────┴────────────────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "patient_visit_encoding['dx_name___diagnosis_code'] = encode_two_columns(data,'dx_name','diagnosis_code')\n",
    "for col in cols_to_encode:\n",
    "    data, patient_visit_encoding[col] = encode_column_and_replace(data,col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>mrn</th><th>diagnosis_code</th><th>dx_name</th><th>dx_name_encoded</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>u64</td></tr></thead><tbody><tr><td>&quot;fa149b1b36855b…</td><td>&quot;ad8291a53a44f0…</td><td>&quot;153.3&quot;</td><td>&quot;Malignant neop…</td><td>4008</td></tr><tr><td>&quot;5e2775d183fc11…</td><td>&quot;5f3940d268bee2…</td><td>&quot;153.3&quot;</td><td>&quot;Malignant neop…</td><td>4008</td></tr><tr><td>&quot;1cb372141215f6…</td><td>&quot;3668823554b94e…</td><td>&quot;188.0&quot;</td><td>&quot;Malignant neop…</td><td>4583</td></tr><tr><td>&quot;f091dc2a1a7965…</td><td>&quot;9736aa7fdd136c…</td><td>&quot;188.0&quot;</td><td>&quot;Malignant neop…</td><td>4583</td></tr><tr><td>&quot;0c357a71be7d7d…</td><td>&quot;c09ed71d87105c…</td><td>&quot;188.0&quot;</td><td>&quot;Malignant neop…</td><td>4583</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌──────────────────┬──────────────────┬────────────────┬───────────────────────────────────┬─────────────────┐\n",
       "│ LOG_ID           ┆ mrn              ┆ diagnosis_code ┆ dx_name                           ┆ dx_name_encoded │\n",
       "│ ---              ┆ ---              ┆ ---            ┆ ---                               ┆ ---             │\n",
       "│ str              ┆ str              ┆ str            ┆ str                               ┆ u64             │\n",
       "╞══════════════════╪══════════════════╪════════════════╪═══════════════════════════════════╪═════════════════╡\n",
       "│ fa149b1b36855b20 ┆ ad8291a53a44f099 ┆ 153.3          ┆ Malignant neoplasm of sigmoid co… ┆ 4008            │\n",
       "│ 5e2775d183fc11d3 ┆ 5f3940d268bee23e ┆ 153.3          ┆ Malignant neoplasm of sigmoid co… ┆ 4008            │\n",
       "│ 1cb372141215f6e2 ┆ 3668823554b94ea1 ┆ 188.0          ┆ Malignant neoplasm of trigone of… ┆ 4583            │\n",
       "│ f091dc2a1a796519 ┆ 9736aa7fdd136c53 ┆ 188.0          ┆ Malignant neoplasm of trigone of… ┆ 4583            │\n",
       "│ 0c357a71be7d7d13 ┆ c09ed71d87105ca5 ┆ 188.0          ┆ Malignant neoplasm of trigone of… ┆ 4583            │\n",
       "└──────────────────┴──────────────────┴────────────────┴───────────────────────────────────┴─────────────────┘"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>mrn</th><th>diagnosis_code</th><th>dx_name</th><th>dx_name_encoded</th></tr><tr><td>u32</td><td>u32</td><td>u32</td><td>u32</td><td>u32</td></tr></thead><tbody><tr><td>0</td><td>0</td><td>38138</td><td>0</td><td>0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 5)\n",
       "┌────────┬─────┬────────────────┬─────────┬─────────────────┐\n",
       "│ LOG_ID ┆ mrn ┆ diagnosis_code ┆ dx_name ┆ dx_name_encoded │\n",
       "│ ---    ┆ --- ┆ ---            ┆ ---     ┆ ---             │\n",
       "│ u32    ┆ u32 ┆ u32            ┆ u32     ┆ u32             │\n",
       "╞════════╪═════╪════════════════╪═════════╪═════════════════╡\n",
       "│ 0      ┆ 0   ┆ 38138          ┆ 0       ┆ 0               │\n",
       "└────────┴─────┴────────────────┴─────────┴─────────────────┘"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.null_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.select(pl.col('LOG_ID'),pl.col('mrn').alias('MRN'),pl.col('dx_name_encoded'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(131455, 3)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.write_csv('EPIC_EMR_cleaned/patient_visit_cleaned.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cleaning patient procedure events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "patient_procedure_event_encoding = {}\n",
    "cols_to_encode = ['EVENT_DISPLAY_NAME'] \n",
    "data = pl.read_csv('EPIC_EMR/patient_procedure events.csv')\n",
    "data = data.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>EVENT_DISPLAY_NAME</th><th>EVENT_TIME</th><th>NOTE_TEXT</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td></tr></thead><tbody><tr><td>&quot;e1a066d449ed18…</td><td>&quot;499386c1a8693f…</td><td>&quot;Quick Note&quot;</td><td>&quot;4/27/19 3:04&quot;</td><td>null</td></tr><tr><td>&quot;068b548cf0d919…</td><td>&quot;68d58e4c2dcd06…</td><td>&quot;Quick Note&quot;</td><td>&quot;5/6/19 11:46&quot;</td><td>null</td></tr><tr><td>&quot;05195f76452108…</td><td>&quot;85a062b39530bf…</td><td>&quot;Transported to…</td><td>&quot;6/12/19 20:59&quot;</td><td>null</td></tr><tr><td>&quot;33a907e9affc52…</td><td>&quot;f7bf5c74f687a5…</td><td>&quot;Mark Now&quot;</td><td>&quot;9/23/19 7:57&quot;</td><td>null</td></tr><tr><td>&quot;59dc47ed0b52c5…</td><td>&quot;424d7ceb87a7de…</td><td>&quot;IV Antibiotics…</td><td>&quot;9/3/19 7:24&quot;</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 5)\n",
       "┌──────────────────┬──────────────────┬───────────────────────────────────┬───────────────┬───────────┐\n",
       "│ LOG_ID           ┆ MRN              ┆ EVENT_DISPLAY_NAME                ┆ EVENT_TIME    ┆ NOTE_TEXT │\n",
       "│ ---              ┆ ---              ┆ ---                               ┆ ---           ┆ ---       │\n",
       "│ str              ┆ str              ┆ str                               ┆ str           ┆ str       │\n",
       "╞══════════════════╪══════════════════╪═══════════════════════════════════╪═══════════════╪═══════════╡\n",
       "│ e1a066d449ed18b0 ┆ 499386c1a8693fd0 ┆ Quick Note                        ┆ 4/27/19 3:04  ┆ null      │\n",
       "│ 068b548cf0d91976 ┆ 68d58e4c2dcd06f6 ┆ Quick Note                        ┆ 5/6/19 11:46  ┆ null      │\n",
       "│ 05195f764521086f ┆ 85a062b39530bf42 ┆ Transported to PACU/ICU with O2,… ┆ 6/12/19 20:59 ┆ null      │\n",
       "│ 33a907e9affc529f ┆ f7bf5c74f687a530 ┆ Mark Now                          ┆ 9/23/19 7:57  ┆ null      │\n",
       "│ 59dc47ed0b52c5d8 ┆ 424d7ceb87a7defd ┆ IV Antibiotics                    ┆ 9/3/19 7:24   ┆ null      │\n",
       "└──────────────────┴──────────────────┴───────────────────────────────────┴───────────────┴───────────┘"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in cols_to_encode:\n",
    "    data, patient_procedure_event_encoding[col] = encode_column_and_replace(data,col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 6)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LOG_ID</th><th>MRN</th><th>EVENT_DISPLAY_NAME</th><th>EVENT_TIME</th><th>NOTE_TEXT</th><th>EVENT_DISPLAY_NAME_encoded</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>u64</td></tr></thead><tbody><tr><td>&quot;e1a066d449ed18…</td><td>&quot;499386c1a8693f…</td><td>&quot;Quick Note&quot;</td><td>&quot;4/27/19 3:04&quot;</td><td>null</td><td>72</td></tr><tr><td>&quot;068b548cf0d919…</td><td>&quot;68d58e4c2dcd06…</td><td>&quot;Quick Note&quot;</td><td>&quot;5/6/19 11:46&quot;</td><td>null</td><td>72</td></tr><tr><td>&quot;05195f76452108…</td><td>&quot;85a062b39530bf…</td><td>&quot;Transported to…</td><td>&quot;6/12/19 20:59&quot;</td><td>null</td><td>27</td></tr><tr><td>&quot;33a907e9affc52…</td><td>&quot;f7bf5c74f687a5…</td><td>&quot;Mark Now&quot;</td><td>&quot;9/23/19 7:57&quot;</td><td>null</td><td>55</td></tr><tr><td>&quot;59dc47ed0b52c5…</td><td>&quot;424d7ceb87a7de…</td><td>&quot;IV Antibiotics…</td><td>&quot;9/3/19 7:24&quot;</td><td>null</td><td>29</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 6)\n",
       "┌──────────────────┬──────────────────┬──────────────────────────┬───────────────┬───────────┬─────────────────────────┐\n",
       "│ LOG_ID           ┆ MRN              ┆ EVENT_DISPLAY_NAME       ┆ EVENT_TIME    ┆ NOTE_TEXT ┆ EVENT_DISPLAY_NAME_enco │\n",
       "│ ---              ┆ ---              ┆ ---                      ┆ ---           ┆ ---       ┆ ded                     │\n",
       "│ str              ┆ str              ┆ str                      ┆ str           ┆ str       ┆ ---                     │\n",
       "│                  ┆                  ┆                          ┆               ┆           ┆ u64                     │\n",
       "╞══════════════════╪══════════════════╪══════════════════════════╪═══════════════╪═══════════╪═════════════════════════╡\n",
       "│ e1a066d449ed18b0 ┆ 499386c1a8693fd0 ┆ Quick Note               ┆ 4/27/19 3:04  ┆ null      ┆ 72                      │\n",
       "│ 068b548cf0d91976 ┆ 68d58e4c2dcd06f6 ┆ Quick Note               ┆ 5/6/19 11:46  ┆ null      ┆ 72                      │\n",
       "│ 05195f764521086f ┆ 85a062b39530bf42 ┆ Transported to PACU/ICU  ┆ 6/12/19 20:59 ┆ null      ┆ 27                      │\n",
       "│                  ┆                  ┆ with O2,…                ┆               ┆           ┆                         │\n",
       "│ 33a907e9affc529f ┆ f7bf5c74f687a530 ┆ Mark Now                 ┆ 9/23/19 7:57  ┆ null      ┆ 55                      │\n",
       "│ 59dc47ed0b52c5d8 ┆ 424d7ceb87a7defd ┆ IV Antibiotics           ┆ 9/3/19 7:24   ┆ null      ┆ 29                      │\n",
       "└──────────────────┴──────────────────┴──────────────────────────┴───────────────┴───────────┴─────────────────────────┘"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.select(['LOG_ID','MRN','EVENT_TIME','EVENT_DISPLAY_NAME_encoded'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.write_csv('EPIC_EMR_cleaned/patient_procedure_events_cleaned.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine all the encoded data from each dataset and store it in json file, inorder to access the actual value of each encoded value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding_dict = {}\n",
    "encoding_dict['patient_history'] = patient_history_encoding\n",
    "encoding_dict['patient_information'] = patient_information_encoding\n",
    "encoding_dict['patient_lab'] = patient_lab_encoding\n",
    "encoding_dict['patient_post_op_complications'] = patient_post_op_encoding\n",
    "encoding_dict['patient_visit'] = patient_visit_encoding\n",
    "encoding_dict['patient_procedure_event'] = patient_procedure_event_encoding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open ('EPIC_EMR_cleaned/encoding.json','w') as f:\n",
    "    json.dump(encoding_dict,f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}