code/01_raw_signals_to_dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import ast\n",
    "import wfdb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Section 1: Converting the raw signals for processing\n",
    "\n",
    "**Note: Do not run this notebook unless you have put the PTB-XL database with the folder 'records100' in a folder named 'ptb_xl' under the 'data' folder. If you place the data anywhere else, modify the 'path' variable.**\n",
    "\n",
    "This notebook converts the raw ECG signals, which are accessed as a 3-dimensional np.array, into \n",
    "\n",
    "1. A 3-dimensional np.array that is saved as type float16 to be fed into MTEX-CNN.\n",
    "\n",
    "2. A dataframe with shape (21837000, 12) that is saved as a compressed gz folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#If sampling rate==100, load the low rate data, and if it's 500, load the high rate data\n",
    "#Note this project only the 100Hz one.\n",
    "def load_raw_data(df, sampling_rate, path):\n",
    "    if sampling_rate == 100:\n",
    "        data = [wfdb.rdsamp(path+f) for f in df.filename_lr]\n",
    "    else:\n",
    "        data = [wfdb.rdsamp(path+f) for f in df.filename_hr]\n",
    "#Get the signal information only\n",
    "    data = np.array([signal for signal, meta in data])\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set path and sampling rate\n",
    "path = '../data/ptb_xl/'\n",
    "sampling_rate=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load and convert annotation data\n",
    "meta_data_df = pd.read_csv(path+'ptbxl_database.csv', index_col='ecg_id')\n",
    "meta_data_df.scp_codes = meta_data_df.scp_codes.apply(lambda x: ast.literal_eval(x))\n",
    "\n",
    "# Load raw signal data\n",
    "raw_signals = load_raw_data(meta_data_df, sampling_rate, path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the raw_signals from float64 to float16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_signals = raw_signals.astype(np.float16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's save the raw_signals both as an np.array and as a dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez('../data/created_data_files/ptb_raw_signals', raw_signals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Rename the columns\n",
    "names = ['patient', 'time', 'lead']\n",
    "\n",
    "#Set a MultiIndex\n",
    "index = pd.MultiIndex.from_product([range(s)for s in raw_signals.shape], names=names)\n",
    "\n",
    "#Save it to a datframe\n",
    "df = pd.DataFrame({'raw_signals': raw_signals.flatten()}, index=index)['raw_signals']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "patient  time  lead\n",
       "0        0     0      -0.119\n",
       "               1      -0.055\n",
       "               2       0.064\n",
       "               3       0.086\n",
       "               4      -0.091\n",
       "                       ...  \n",
       "21836    999   7       0.291\n",
       "               8       0.178\n",
       "               9       0.106\n",
       "               10      0.047\n",
       "               11     -0.103\n",
       "Name: raw_signals, Length: 262044000, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check df\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Unstack and sort index\n",
    "df = df.unstack(level='lead').sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lead</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>patient</th>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">21836</th>\n",
       "      <th>995</th>\n",
       "      <td>-0.024</td>\n",
       "      <td>-0.016</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.019</td>\n",
       "      <td>-0.016</td>\n",
       "      <td>-0.003</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.024</td>\n",
       "      <td>-0.012</td>\n",
       "      <td>-0.044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>-0.002</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.001</td>\n",
       "      <td>-0.001</td>\n",
       "      <td>0.071</td>\n",
       "      <td>0.148</td>\n",
       "      <td>0.082</td>\n",
       "      <td>0.034</td>\n",
       "      <td>-0.009</td>\n",
       "      <td>-0.085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.091</td>\n",
       "      <td>0.012</td>\n",
       "      <td>-0.079</td>\n",
       "      <td>-0.051</td>\n",
       "      <td>0.085</td>\n",
       "      <td>-0.034</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.199</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.063</td>\n",
       "      <td>0.016</td>\n",
       "      <td>-0.117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.175</td>\n",
       "      <td>0.022</td>\n",
       "      <td>-0.153</td>\n",
       "      <td>-0.099</td>\n",
       "      <td>0.164</td>\n",
       "      <td>-0.065</td>\n",
       "      <td>0.070</td>\n",
       "      <td>0.244</td>\n",
       "      <td>0.148</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.018</td>\n",
       "      <td>-0.108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.166</td>\n",
       "      <td>-0.007</td>\n",
       "      <td>-0.173</td>\n",
       "      <td>-0.080</td>\n",
       "      <td>0.170</td>\n",
       "      <td>-0.090</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.291</td>\n",
       "      <td>0.178</td>\n",
       "      <td>0.106</td>\n",
       "      <td>0.047</td>\n",
       "      <td>-0.103</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "lead             0      1      2      3      4      5      6      7      8   \\\n",
       "patient time                                                                  \n",
       "21836   995  -0.024 -0.016  0.008  0.019 -0.016 -0.003  0.069  0.135  0.073   \n",
       "        996   0.000 -0.001 -0.002  0.001  0.001 -0.001  0.071  0.148  0.082   \n",
       "        997   0.091  0.012 -0.079 -0.051  0.085 -0.034  0.069  0.199  0.120   \n",
       "        998   0.175  0.022 -0.153 -0.099  0.164 -0.065  0.070  0.244  0.148   \n",
       "        999   0.166 -0.007 -0.173 -0.080  0.170 -0.090  0.069  0.291  0.178   \n",
       "\n",
       "lead             9      10     11  \n",
       "patient time                       \n",
       "21836   995   0.024 -0.012 -0.044  \n",
       "        996   0.034 -0.009 -0.085  \n",
       "        997   0.063  0.016 -0.117  \n",
       "        998   0.080  0.018 -0.108  \n",
       "        999   0.106  0.047 -0.103  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check df again\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21837000, 12)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check shape\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save to csv as a compressed gz folder\n",
    "df.to_csv('../data/created_data_files/ptb_signals_df.csv.gz', compression='gzip', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}