14065textClassification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "#from sklearn.datasets import load_files  \n",
    "#nltk.download('stopwords') \n",
    "#import pickle \n",
    "  \n",
    "trainData = pd.read_csv(\"C:/Users/Mahendra/Desktop/lab/trainset.txt\", sep='\\t')\n",
    "testData = pd.read_csv(\"C:/Users/Mahendra/Desktop/lab/testsetwithoutlabels.txt\", sep='\\t')\n",
    "trainBody = trainData['TITLE'].values + trainData['BODY'].values\n",
    "testBody = testData['TITLE'].values + testData['BODY'].values\n",
    "Y_train = trainData['CLASS'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Mahendra\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\Mahendra\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Mahendra\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Mahendra\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import re  \n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords') \n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "def get_wordnet_pos(word):\n",
    "    tag = nltk.pos_tag([word])[0][1][0].upper()\n",
    "    tag_dict = {\"J\": wordnet.ADJ,\n",
    "                \"N\": wordnet.NOUN,\n",
    "                \"V\": wordnet.VERB,\n",
    "                \"R\": wordnet.ADV}\n",
    "\n",
    "    return tag_dict.get(tag, wordnet.NOUN)\n",
    "\n",
    "def preprocessText(Text):\n",
    "    processedText = []\n",
    "\n",
    "    lemmatizer = WordNetLemmatizer() \n",
    "\n",
    "    for paragraph in range(0, len(Text)):  \n",
    "        # Substituting all the special characters with single space\n",
    "        para = re.sub(r'\\W', ' ', str(Text[paragraph]))\n",
    "\n",
    "        # Substituting all single characters with single space\n",
    "        para = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', para)\n",
    "\n",
    "        # Substituting single characters from the start\n",
    "        para = re.sub(r'\\^[a-zA-Z]\\s+', ' ', para)\n",
    "        \n",
    "        # Substituting numbers with a space\n",
    "        para = re.sub(r'\\d+', ' ', para)\n",
    "    \n",
    "        # Substituting single characters from the start\n",
    "        para = re.sub(r'\\^[a-zA-Z]\\s+', ' ', para) \n",
    "\n",
    "        # Substituting multiple spaces with single space\n",
    "        para = re.sub(r'\\s+', ' ', para, flags=re.I)\n",
    "\n",
    "        # Converting to Lowercase\n",
    "        para = para.lower()\n",
    "\n",
    "        # split every word from paragraph \n",
    "        para = para.split()\n",
    "\n",
    "        #removing stopwords\n",
    "        stop_words = set(stopwords.words('english'))\n",
    "        for w in para: \n",
    "            if w in stop_words: \n",
    "                para.remove(w)\n",
    "        \n",
    "        # Lemmatization\n",
    "        para = [lemmatizer.lemmatize(word, get_wordnet_pos(word)) for word in para]\n",
    "        para = ' '.join(para)\n",
    "\n",
    "        processedText.append(para)\n",
    "    return processedText\n",
    "\n",
    "X_train = preprocessText(trainBody)\n",
    "X_test = preprocessText(testBody)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, -1, 1, -1, -1, -1, -1, 1, 1, -1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1, 1, -1, 1, 1, -1, -1, -1, -1, 1, -1, 1, -1, 1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, 1, 1, 1, 1, -1, 1, 1, -1, -1, 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1, 1, -1, 1, 1, -1, 1]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_files  \n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from numpy.core.umath_tests import inner1d\n",
    "\n",
    "\n",
    "def BagofWords(trainData,testData):\n",
    "    vectorizer = CountVectorizer(analyzer='word', input='content', stop_words='english')\n",
    "    bag_of_words_train = vectorizer.fit_transform(trainData).toarray()\n",
    "    bag_of_words_test = vectorizer.transform(testData).toarray()\n",
    "    return bag_of_words_train,bag_of_words_test\n",
    "\n",
    "def Tfidf(trainData,testData):\n",
    "    vectorizer = CountVectorizer(analyzer='word', input='content', stop_words='english')\n",
    "    tfidf_train  = vectorizer.fit_transform(trainData).toarray()\n",
    "    tfidf_test  = vectorizer.transform(testData).toarray()\n",
    "    return tfidf_train, tfidf_test \n",
    "\n",
    "train_X1,test_X1 = BagofWords(X_train, X_test)\n",
    "train_X2,test_X2 = Tfidf(X_train, X_test)\n",
    "\n",
    "classifier = RandomForestClassifier()\n",
    "classifier.fit(train_X2, Y_train)\n",
    "\n",
    "prediction = classifier.predict(test_X2).tolist()\n",
    "# prediction2 = classifier.predict(test_X2).tolist()\n",
    "\n",
    "print(prediction)\n",
    "\n",
    "# file = open(\"e14065.txt\",\"w\") \n",
    " \n",
    "# for label in prediction:\n",
    "#     if(label==1):\n",
    "#         file.write(\"+%s\\n\" % label)\n",
    "        \n",
    "#     else:\n",
    "#         file.write(\"%s\\n\" % label)\n",
    "        \n",
    " \n",
    "# file.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}