code_classification_fall2018.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "###### INCLUSION CRITERIA ANALYSIS###\n",
    "####The below code \n",
    "#Creators: Xiaoru Dong\n",
    "#Contributors:\n",
    "#Last updated: 12/10/2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read the inclusion criteria data set\n",
    "file = '/Users/ruby/Downloads/Inclusion_Criteria_Annotation.csv'\n",
    "xl = pd.ExcelFile(file)\n",
    "df = xl.parse('All Inclusion Criteria (5420)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The list of stop words\n",
    "stop_words = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \n",
    "              \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", \n",
    "              'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', \n",
    "              'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', \n",
    "              'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', \n",
    "              'the', 'if', 'because', 'as', 'until', 'of', 'at', 'by', 'for', 'with', 'about', 'between', 'into',\n",
    "              'through', 'during', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', \n",
    "              'under', 'again', 'further', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'few', 'more', \n",
    "              'most', 'some', 'such', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', \n",
    "              'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lineNum = len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set a function for outputing features\n",
    "def writefeatures(path, features):\n",
    "    with open(path,\"w\") as fp: \n",
    "        for word in features:\n",
    "            fp.write(word + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set a function to get an arff file used as the input in Weka\n",
    "def writewekainput(path, features, lineNum, eachline, rct):\n",
    "    with open(path,\"w\") as fp:        #this line of code to create a new file\n",
    "        fp.write('''@RELATION classification_analysis\\n''')\n",
    "    \n",
    "        for i in range(len(features)): \n",
    "            fp.write('''@ATTRIBUTE ''' + \"word\" + str(i) + ''' NUMERIC\\n''') \n",
    "    \n",
    "        fp.write('''@ATTRIBUTE class {1.0, 0.0}\\n''')\n",
    "        fp.write('''@DATA\\n''')\n",
    "\n",
    "#this code is to check if a word appear in the feature list or not. If yes, then write \"1\", if no then write \"0\"\n",
    "    \n",
    "        for i in range(lineNum): \n",
    "            binary = {}\n",
    "            for word in features:\n",
    "                if word in eachline[i]:\n",
    "                    binary[word] = 1\n",
    "                else:\n",
    "                    binary[word] = 0\n",
    "                fp.write(str(binary[word]) + \",\")\n",
    "            fp.write(rct[i] + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Extract Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results1 = []\n",
    "wordlist = []\n",
    "rct = []\n",
    "eachline = []\n",
    "\n",
    "# split the sentence into words and save in a list\n",
    "for i in range(lineNum): \n",
    "    \n",
    "    string_rct = str(df['Only RCTs'][i])\n",
    "    if string_rct == 'nan':\n",
    "        string_rct = '0.0'\n",
    "    if string_rct == 'x':\n",
    "        string_rct = '1.0'\n",
    "    rct.append(string_rct)\n",
    "    \n",
    "    string = str(df['Inclusion Criteria'][i]).lower().strip()\n",
    "    wordlist = string.split(' ')\n",
    "    \n",
    "    for j in range(len(wordlist)):\n",
    "        wordlist[j] = wordlist[j].replace('.', '')\n",
    "        wordlist[j] = wordlist[j].replace(')', '')\n",
    "        wordlist[j] = wordlist[j].replace('(', '')\n",
    "        wordlist[j] = wordlist[j].replace(',', '')\n",
    "        wordlist[j] = wordlist[j].replace(\"'\", '')\n",
    "        wordlist[j] = wordlist[j].replace('\"', '')\n",
    "        results1.append(wordlist[j])\n",
    "    eachline.append(wordlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove empty element in the list\n",
    "results1 = list(OrderedDict.fromkeys(results1))\n",
    "results1 = [x for x in results1 if x] \n",
    "\n",
    "# remove stop words\n",
    "results1 = [word for word in results1 if word not in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output features and save in a txt file\n",
    "featureword0_savepath = \"/Users/ruby/Desktop/AllWords.txt\" \n",
    "writefeatures(featureword0_savepath, results1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output the weka input file\n",
    "weka0_path=\"/Users/ruby/Desktop/AllWords.arff\"\n",
    "writewekainput(weka0_path, results1, lineNum, eachline, rct)\n",
    "\n",
    "# We use the weka to read the weka input file. Then, we could use weka to get the results \n",
    "# about precision, recall and F-meaure base in 3 algorithms: Random Forest, J48 and Naïve Bayes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# When we examined features, we found that some words are redundancy. \n",
    "# For example, “quasi-rcts” and “quasi-randomized” both represent the same meaning. \n",
    "# For these kinds of words, we only kept one version. \n",
    "# We implemented this by manually creating a list of the words for Python to replace\n",
    "# We use the following code to get the new features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove same meaning words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results2 = []\n",
    "wordlist2 = []\n",
    "rct = []\n",
    "eachline2 = []\n",
    "\n",
    "#replace the same meaning words\n",
    "quasi = ['quasi-rcts', 'quasi-randomized', 'quasi-randomised', 'quasi-random', 'quasi-randomly']\n",
    "cct = ['cct', 'ccts']\n",
    "cba = ['cba', 'cbas', 'before-after']\n",
    "privative = [\"not\",\"didn't\", \"don't\", \"won't\", \"isn't\", \"aren't\", \"wasn't\", \"weren't\", \"needn't\", \"wouldn't\", \"shouldn't\", \"can't\", \"couldn't\", \"no\"]\n",
    "\n",
    "# split the sentence into words and save in a list\n",
    "for i in range(lineNum): \n",
    "    \n",
    "    string_rct = str(df['Only RCTs'][i])\n",
    "    if string_rct == 'nan':\n",
    "        string_rct = '0.0'\n",
    "    if string_rct == 'x':\n",
    "        string_rct = '1.0'\n",
    "    rct.append(string_rct)\n",
    "    \n",
    "    string = str(df['Inclusion Criteria'][i]).lower().strip()\n",
    "    wordlist2 = string.split(' ')\n",
    "    \n",
    "    for j in range(len(wordlist2)):\n",
    "        wordlist2[j] = wordlist2[j].replace('.', '')\n",
    "        wordlist2[j] = wordlist2[j].replace(')', '')\n",
    "        wordlist2[j] = wordlist2[j].replace('(', '')\n",
    "        wordlist2[j] = wordlist2[j].replace(',', '')\n",
    "        wordlist2[j] = wordlist2[j].replace(\"'\", '')\n",
    "        wordlist2[j] = wordlist2[j].replace('\"', '')\n",
    "        if wordlist2[j] in quasi:\n",
    "            wordlist2[j] = 'quasi-rcts'\n",
    "        if wordlist2[j] in cct:\n",
    "            wordlist2[j] = 'cct'\n",
    "        if wordlist2[j] in cba:\n",
    "            wordlist2[j] = 'cba'\n",
    "        if wordlist2[j] in privative:\n",
    "            wordlist2[j] = 'not'\n",
    "        results2.append(wordlist2[j])\n",
    "    eachline2.append(wordlist2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2 = list(OrderedDict.fromkeys(results2))\n",
    "results2 = [x for x in results2 if x] #remove empty element in the list\n",
    "results2 = [word for word in results2 if word not in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# output features and save in a txt file\n",
    "featureinfoword_savepath = \"/Users/ruby/Desktop/AllWord_Noredundant.txt\"\n",
    "writefeatures(featureinfoword_savepath, results2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output the weka input file\n",
    "weka_path=\"/Users/ruby/Desktop/AllWord_Noredundant.arff\"\n",
    "writewekainput(weka_path, results2, lineNum, eachline2, rct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We use the weka to read the weka input file. \n",
    "# Then, we could use weka to get words selected by Information Gain. \n",
    "# We save the results of words selected by Information Gain in Weka.\n",
    "# We use the following code to get words selected by Information Gain in Weka and \n",
    "# the weka input file for words selected by Information Gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# infomative words "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(\"/Users/ruby/Desktop/InformativeWords\",\"r\")\n",
    "lines = f.readlines()\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = lines[21:]\n",
    "newlines=[]\n",
    "for string in lines:\n",
    "    if string != \"\\n\":\n",
    "        newlines.append(string)\n",
    "\n",
    "\n",
    "info_codelist = []\n",
    "for i in range(len(newlines) - 1):\n",
    "    string = newlines[i]\n",
    "    if float(string.split(' ')[1]) > 0:\n",
    "        info_codelist.append(int(string.split(' ')[-2])-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read the informative words and save them in a list\n",
    "info_wordlist = []\n",
    "for word in (info_codelist):\n",
    "    info_wordlist.append(results2[word])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output features\n",
    "featureinfoword_savepath = \"/Users/ruby/Desktop/WordsSelectedByInformationGain.txt\"\n",
    "writefeatures(featureinfoword_savepath, info_wordlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output the weka input file\n",
    "wekainfo_path=\"/Users/ruby/Desktop/WordsSelectedByInformationGain.arff\"\n",
    "writewekainput(wekainfo_path, info_wordlist, lineNum, eachline2, rct)\n",
    "\n",
    "# We use the weka to read the weka input file. Then, we could use weka to get the results \n",
    "# about precision, recall and F-meaure base in 3 algorithms: Random Forest, J48 and Naïve Bayes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We manually checked the list of informative words to analyze whether or not \n",
    "# the new classification model was over-fitting. \n",
    "# We found that some words after feature selection using information gain \n",
    "# were domain-specific words, numbers, and signs. \n",
    "# Because we want to build a classifier that could be used in the whole medical field, \n",
    "# we removed the domain-specific words, numbers, and signs manually.\n",
    "# The following code is used to get the weka input file for manual analysis words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Manual Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read the manual analysis data set\n",
    "file = '/Users/ruby/Downloads/FeatureExtraction_FeatureSelection.csv'\n",
    "x3 = pd.ExcelFile(file)\n",
    "df4 = x3.parse('Manual Analysis Words (407)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read the informative words and save them in a list\n",
    "info_wordlist = []\n",
    "for word in df4['Word']:\n",
    "    info_wordlist.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output the weka input file\n",
    "wekainfo3_path=\"/Users/ruby/Desktop/manualanalysis_words.arff\"\n",
    "writewekainput(wekainfo3_path, info_wordlist, lineNum, eachline2, rct)\n",
    "\n",
    "# We use the weka to read the weka input file. Then, we could use weka to get the results \n",
    "# about precision, recall and F-meaure base in 3 algorithms: Random Forest, J48 and Naïve Bayes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}