information_gain_feature_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "###### INCLUSION CRITERIA ANALYSIS - INFORMATION GAIN FEATURES SELECTION###\n",
    "####The below code \n",
    "#Creators: Xiaoru Dong\n",
    "#Contributors: Linh Hoang\n",
    "#Last updated: 04/8/2019\n",
    "#The below Python script is used to generate features and to create the Weka input files \n",
    "#corresponding to the information gain feature selection approach that we implemented in the study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "######GLOBAL FUNCTIONS TO EXTRACT FEATURE AND TO GENERATE WEKA INPUT FILES####\n",
    "\n",
    "#Set a function for outputing features\n",
    "def writefeatures(path, features):\n",
    "    with open(path,\"w\") as fp: \n",
    "        for word in features:\n",
    "            fp.write(word + \"\\n\")\n",
    "            \n",
    "#Set a function to get an arff file used as the input in Weka\n",
    "def writewekainput(path, features, lineNum, eachline, rct):\n",
    "    with open(path,\"w\") as fp:        #this line of code to create a new file\n",
    "        fp.write('''@RELATION classification_analysis\\n''')\n",
    "    \n",
    "        for i in range(len(features)): \n",
    "            fp.write('''@ATTRIBUTE ''' + \"word\" + str(i) + ''' NUMERIC\\n''') \n",
    "    \n",
    "        fp.write('''@ATTRIBUTE class {1.0, 0.0}\\n''')\n",
    "        fp.write('''@DATA\\n''')\n",
    "\n",
    "        #check if a word appear in the feature list or not. If yes, then write \"1\", if no then write \"0\"\n",
    "        for i in range(lineNum): \n",
    "            binary = {}\n",
    "            for word in features:\n",
    "                if word in eachline[i]:\n",
    "                    binary[word] = 1\n",
    "                else:\n",
    "                    binary[word] = 0\n",
    "                fp.write(str(binary[word]) + \",\")\n",
    "            fp.write(rct[i] + \"\\n\")\n",
    "\n",
    "#Get the path of the folder that contains the annotation file\n",
    "#Change this path to the actual path of your local machine\n",
    "path = \"/Users/phuong/Downloads/Inclusion_Criteria_Revised/\"\n",
    "\n",
    "#Read the inclusion criteria annotation file\n",
    "file = path + 'Inclusion_Criteria_Annotation.csv'\n",
    "df = pd.read_csv(file, encoding = \"ISO-8859-1\")\n",
    "lineNum = len(df)\n",
    "\n",
    "#The list of stop words\n",
    "stop_words = ['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \n",
    "              \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", \n",
    "              'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', \n",
    "              'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', \n",
    "              'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', \n",
    "              'the', 'if', 'because', 'as', 'until', 'of', 'at', 'by', 'for', 'with', 'about', 'between', 'into',\n",
    "              'through', 'during', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', \n",
    "              'under', 'again', 'further', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'few', 'more', \n",
    "              'most', 'some', 'such', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', \n",
    "              'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove same meaning words\n",
    "allwords_noredundant_features = []\n",
    "wordlist_noredundant_oneline = []\n",
    "rct = []\n",
    "wordlist_noredundant_alllines = []\n",
    "\n",
    "#replace the same meaning words\n",
    "quasi = ['quasi-rcts', 'quasi-randomized', 'quasi-randomised', 'quasi-random', 'quasi-randomly']\n",
    "cct = ['cct', 'ccts']\n",
    "cba = ['cba', 'cbas', 'before-after']\n",
    "privative = [\"not\",\"didn't\", \"don't\", \"won't\", \"isn't\", \"aren't\", \"wasn't\", \"weren't\", \"needn't\", \"wouldn't\", \"shouldn't\", \"can't\", \"couldn't\", \"no\"]\n",
    "\n",
    "# split the sentence into words and save in a list\n",
    "for i in range(lineNum): \n",
    "    \n",
    "    string_rct = str(df['Only RCTs'][i])\n",
    "    if string_rct == 'nan':\n",
    "        string_rct = '0.0'\n",
    "    if string_rct == 'x':\n",
    "        string_rct = '1.0'\n",
    "    rct.append(string_rct)\n",
    "    \n",
    "    string = str(df['Inclusion Criteria'][i]).lower().strip()\n",
    "    wordlist_noredundant_oneline = string.split(' ')\n",
    "    \n",
    "    for j in range(len(wordlist_noredundant_oneline)):\n",
    "        wordlist_noredundant_oneline[j] = wordlist_noredundant_oneline[j].replace('.', '')\n",
    "        wordlist_noredundant_oneline[j] = wordlist_noredundant_oneline[j].replace(')', '')\n",
    "        wordlist_noredundant_oneline[j] = wordlist_noredundant_oneline[j].replace('(', '')\n",
    "        wordlist_noredundant_oneline[j] = wordlist_noredundant_oneline[j].replace(',', '')\n",
    "        wordlist_noredundant_oneline[j] = wordlist_noredundant_oneline[j].replace(\"'\", '')\n",
    "        wordlist_noredundant_oneline[j] = wordlist_noredundant_oneline[j].replace('\"', '')\n",
    "        if wordlist_noredundant_oneline[j] in quasi:\n",
    "            wordlist_noredundant_oneline[j] = 'quasi-rcts'\n",
    "        if wordlist_noredundant_oneline[j] in cct:\n",
    "            wordlist_noredundant_oneline[j] = 'cct'\n",
    "        if wordlist_noredundant_oneline[j] in cba:\n",
    "            wordlist_noredundant_oneline[j] = 'cba'\n",
    "        if wordlist_noredundant_oneline[j] in privative:\n",
    "            wordlist_noredundant_oneline[j] = 'not'\n",
    "        allwords_noredundant_features.append(wordlist_noredundant_oneline[j])\n",
    "    wordlist_noredundant_alllines.append(wordlist_noredundant_oneline)\n",
    "    \n",
    "allwords_noredundant_features = list(OrderedDict.fromkeys(allwords_noredundant_features))\n",
    "allwords_noredundant_features = [x for x in allwords_noredundant_features if x] #remove empty element in the list\n",
    "allwords_noredundant_features = [word for word in allwords_noredundant_features if word not in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(path + \"InformativeWords\",\"r\")\n",
    "lines = f.readlines()\n",
    "f.close()\n",
    "\n",
    "lines = lines[21:]\n",
    "newlines=[]\n",
    "for string in lines:\n",
    "    if string != \"\\n\":\n",
    "        newlines.append(string)\n",
    "\n",
    "\n",
    "info_codelist = []\n",
    "for i in range(len(newlines) - 1):\n",
    "    string = newlines[i]\n",
    "    if float(string.split(' ')[1]) > 0:\n",
    "        info_codelist.append(int(string.split(' ')[-2])-1)\n",
    "        \n",
    "\n",
    "# read the informative words and save them in a list\n",
    "informationgain_features = []\n",
    "for word in (info_codelist):\n",
    "    informationgain_features.append(allwords_noredundant_features[word])\n",
    "    \n",
    "# output features\n",
    "informationgain_features_savepath = path + \"WordsSelectedByInformationGain.csv\"\n",
    "writefeatures(informationgain_features_savepath, informationgain_features)\n",
    "\n",
    "# output the weka input file\n",
    "wekainput_informationgain_path= path + \"WordsSelectedByInformationGain_weka_input.arff\"\n",
    "writewekainput(wekainput_informationgain_path, informationgain_features, lineNum, wordlist_noredundant_alllines, rct)\n",
    "\n",
    "# We use the weka to read the weka input file. Then, we could use weka to get the results \n",
    "# about precision, recall and F-meaure base in 3 algorithms: Random Forest, J48 and Naïve Bayes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}