bigram.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Import Data\n",
    "- pos_cls is the medical data\n",
    "- neg_cls is the non-medical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = './data/'\n",
    "MEDICAL_CSV = DATA_DIR + 'icd_10_2017.csv'\n",
    "NON_MEDICAL = DATA_DIR + 'big.txt'\n",
    "UMLS_CSV = DATA_DIR + 'umls.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_icd = pd.read_csv(MEDICAL_CSV, header=None, usecols=[3, 4])\n",
    "pos_cls = df_icd[4].tolist()\n",
    "print(\"%d lines in pos_cls data.\" % len(pos_cls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df_umls = pd.read_csv(UMLS_CSV, sep='***', delimiter='\\n\\r', header=None)\n",
    "umls = [_.strip('\"') for _ in df_umls[0].tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pos_cls = umls + pos_cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(\"%d lines in pos_cls data.\" % len(pos_cls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with open(NON_MEDICAL, encoding=\"utf-8\") as file:\n",
    "    neg_cls = [_.strip() for _ in \" \".join([l.strip() for l in file]).split(\".\")]\n",
    "print(\"%d lines in neg_cls data.\" % len(neg_cls))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Preprocessing\n",
    "\n",
    "```https://github.com/shams-sam/logic-lab/blob/master/TextPreprocessing/__preprocessing.py```\n",
    "\n",
    "- using the standard code for preprocessing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from preprocessing import text_preprocessing\n",
    "pre = partial(text_preprocessing, HYPHEN_HANDLE = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pos_cls = [pre(_) for _ in pos_cls]\n",
    "neg_cls = [pre(_) for _ in neg_cls]\n",
    "print(\"%d lines in pos_cls data.\" % len(pos_cls))\n",
    "print(\"%d lines in neg_cls data.\" % len(neg_cls))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "import numpy as np\n",
    "from sklearn.utils import class_weight\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "NGRAM = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_tokenizer = Tokenizer()\n",
    "data_tokenizer.fit_on_texts(pos_cls + neg_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_index = {v: k for k, v in data_tokenizer.word_index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pos_seq = data_tokenizer.texts_to_sequences(pos_cls)\n",
    "neg_seq = data_tokenizer.texts_to_sequences(neg_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "padding = [0] * (NGRAM-1)\n",
    "pos_seq = [padding + _ + padding for _ in pos_seq]\n",
    "neg_seq = [padding + _ + padding for _ in neg_seq]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "cls_val = 0\n",
    "for _ in [neg_seq, pos_seq]:\n",
    "    for __ in _:\n",
    "        for idx in range(0, len(__)-NGRAM+1):\n",
    "            X.append(__[idx: idx+NGRAM])\n",
    "            y.append(cls_val)\n",
    "    cls_val += 1\n",
    "assert len(X) == len(y)\n",
    "num_pos_cls = len([_ for _ in y if _ == 1])\n",
    "num_neg_cls = len([_ for _ in y if _ == 0])\n",
    "assert num_pos_cls + num_neg_cls == len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(\"%d training data available.\" % len(X))\n",
    "print(\"%d positive data available.\" % num_pos_cls)\n",
    "print(\"%d negative data available.\" % num_neg_cls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = np.array(X)\n",
    "y = np.array(y)\n",
    "print(\"shape X: %d rows, %d columns\" % X.shape)\n",
    "print(\"shape y: %d rows\" % y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class_weights = class_weight.compute_class_weight('balanced', np.unique(y), y)\n",
    "print('number of classes:', len(class_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "shuffle_split = StratifiedShuffleSplit(n_splits=3, test_size=0.3, random_state=0)\n",
    "shuffle_split.get_n_splits(X, y)\n",
    "for train_index, test_index in shuffle_split.split(X, y):\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "assert len(X_train) == len(y_train)\n",
    "assert len(X_test) == len(y_test)\n",
    "print(\"%d in train set.\" % len(y_train))\n",
    "print(\"%d in test set.\" % len(y_test))\n",
    "weight_val = np.ones(len(y_test))\n",
    "for i in range(len(y_test)):\n",
    "    weight_val[i] *= class_weights[y_test[i]-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "y_train = to_categorical(y_train, num_classes=len(class_weights))\n",
    "y_test = to_categorical(y_test, num_classes=len(class_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(\"shape X_train: %d rows, %d columns \" % X_train.shape)\n",
    "print(\"shape y_train: %d rows, %d columns\" % y_train.shape)\n",
    "print(\"shape X_test: %d rows, %d columns \" % X_test.shape)\n",
    "print(\"shape y_test: %d rows, %d columns\" % y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Word2Vec and Embedding Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "WORD2VEC_MODEL = '/data/Discharge_Summary/Diagnosis_ICD/master/wikipedia-pubmed-and-PMC-w2v.bin'\n",
    "EMBEDDING_DIM = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "w2v_model = gensim.models.KeyedVectors.load_word2vec_format(WORD2VEC_MODEL, binary=True)\n",
    "def embedding_index(word):\n",
    "    return w2v_model.word_vec(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "nb_words = len(data_tokenizer.word_index)+1\n",
    "embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))\n",
    "for word, i in data_tokenizer.word_index.items():\n",
    "    if word in w2v_model.vocab:\n",
    "        embedding_matrix[i] = embedding_index(word)\n",
    "print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "num_lstm = 234\n",
    "num_dense = 142\n",
    "rate_drop_lstm = 0.21\n",
    "rate_drop_dense = 0.24\n",
    "act = 'relu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.models import Model\n",
    "import datetime\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(nb_words,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=NGRAM,\n",
    "        trainable=False)\n",
    "lstm_layer = LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm)\n",
    "\n",
    "sequence_input = Input(shape=(NGRAM,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "x = lstm_layer(embedded_sequences)\n",
    "x = BatchNormalization()(x)\n",
    "x = Dropout(rate_drop_dense)(x)\n",
    "\n",
    "x = Dense(num_dense, activation=act)(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Dropout(rate_drop_dense)(x)\n",
    "\n",
    "preds = Dense(len(class_weights), activation='softmax')(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model = Model(inputs=[sequence_input], \\\n",
    "        outputs=preds)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "        optimizer='adam',\n",
    "        metrics=['acc'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "time = datetime.datetime.now().strftime('D%Y%m%d_T%H%M')\n",
    "STAMP = 'bigram_model_' + str(time) +  '_%d_%d_%.2f_%.2f'%(num_lstm, num_dense, rate_drop_lstm, \\\n",
    "        rate_drop_dense)\n",
    "print(STAMP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "early_stopping =EarlyStopping(monitor='val_loss', patience=3)\n",
    "bst_model_path = STAMP + '.h5'\n",
    "model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "try:\n",
    "    hist = model.fit([X_train], y_train, \\\n",
    "        validation_data=([X_test], y_test, weight_val), \\\n",
    "        epochs=10, batch_size=2048, shuffle=True, \\\n",
    "        class_weight=class_weights, callbacks=[early_stopping, model_checkpoint])\n",
    "except:\n",
    "    print(\"\\n\\nTraining Stopped Manually.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "hist.history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Model Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_index[0] = '***'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_prediction(sentence, verbose = False):\n",
    "    sentence = text_preprocessing(sentence)\n",
    "    seq = data_tokenizer.texts_to_sequences([sentence])\n",
    "    seq = seq[0]\n",
    "    print(seq)\n",
    "    result = []\n",
    "    print(len(seq))\n",
    "    insert_end = False\n",
    "    insert_start = True\n",
    "    for idx in range(0, len(seq)-NGRAM+1):\n",
    "        category = model.predict(np.atleast_2d([seq[idx: idx+NGRAM]]))\n",
    "        cat = category.argmax()\n",
    "        print(data_index[seq[idx]], data_index[seq[idx+1]], cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "get_prediction('patient has diabetes while needs the procedure of deep vein thrombosis. He also has high temperature and other infections')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import _pickle as pkl\n",
    "pkl.dump(data_tokenizer, open('bigram_data_tokenizer.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}