nb_amazonReview.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn import decomposition, ensemble\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "import os\n",
    "from sklearn.externals import joblib \n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class naive_bayes_sentiment:\n",
    "    def data_preproc(self):\n",
    "        \n",
    "        #Loading the data\n",
    "        data = open('/Users/seethu-8363/Downloads/amazon_reviews.csv').read()\n",
    "        \n",
    "        labels, texts = [], []\n",
    "        for i, line in enumerate(data.split(\"\\n\")):\n",
    "            content = line.split()\n",
    "            labels.append(content[0])\n",
    "            texts.append(\" \".join(content[1:]))\n",
    "        # create a dataframe using texts and lables\n",
    "        trainDF = pd.DataFrame()\n",
    "        trainDF['text'] = texts\n",
    "        trainDF['label'] = labels\n",
    "        \n",
    "        #lower case\n",
    "        trainDF[\"text\"] = trainDF[\"text\"].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n",
    "        trainDF[\"text\"].head()\n",
    "        \n",
    "        #STOP WORDS\n",
    "        #Removal of stop words\n",
    "        stop = stopwords.words('english')\n",
    "        ##Creating a list of custom stopwords\n",
    "        new_words = [\"using\", \"show\", \"result\", \"large\", \"also\", \"iv\", \"one\", \"two\", \"new\", \"previously\", \"shown\",\"way\",\"ever\",\n",
    "             \"us\",\"up\",\"cd\",\"that\",\"all\",\"second\"]\n",
    "        stop.append(new_words)\n",
    "        trainDF[\"text\"] = trainDF[\"text\"].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\n",
    "        \n",
    "        \n",
    "        #Removal of punctautaion \n",
    "        trainDF[\"text\"] = trainDF[\"text\"].str.replace('[^\\w\\s]','  ')\n",
    "\n",
    "        \n",
    "        # split the dataset into training and validation datasets \n",
    "        train_x, valid_x, train_y, valid_y = model_selection.train_test_split(trainDF['text'], trainDF['label'])\n",
    "        \n",
    "        # label encode the target variable \n",
    "        encoder = preprocessing.LabelEncoder()\n",
    "        train_y = encoder.fit_transform(train_y)\n",
    "        valid_y = encoder.fit_transform(valid_y)\n",
    "        \n",
    "    \n",
    "        return train_x, train_y, valid_x, valid_y\n",
    "        \n",
    "    def train_model_pipeline(self, classifier, train_x, label , valid_x , valid_y):\n",
    "            # fit the training dataset on the classifier\n",
    "            \n",
    "            tfidf_vectorizer = TfidfVectorizer()\n",
    "            nb = classifier\n",
    "            #creating pipeline for model mand vectoriser\n",
    "            tfidf_nv_pipe = Pipeline([('tfidf', tfidf_vectorizer), ('nb', nb)])\n",
    "\n",
    "            #Fitting the data\n",
    "            tfidf_nv_pipe.fit(train_x,label)\n",
    "    \n",
    "            # predict the labels on validation dataset\n",
    "            predictions = tfidf_nv_pipe.predict(valid_x)\n",
    "    \n",
    "            return metrics.f1_score(predictions, valid_y),tfidf_nv_pipe\n",
    "    \n",
    "    def load_model(self, train_x , train_y , valid_x , valid_y):\n",
    "        \n",
    "        # Naive Bayes on Word Level TF IDF Vectors\n",
    "        accuracy,model = self.train_model_pipeline(naive_bayes.MultinomialNB(), train_x, train_y, valid_x , valid_y)\n",
    "     \n",
    "        #pickle model\n",
    "        joblib.dump(model,current_path+\"/model/sentiment_analysis_naive_bayes_model.pkl\")\n",
    "        print(\"NB, WordLevel TF-IDF ACCURACY: \", accuracy)\n",
    "        \n",
    "        return model\n",
    "    \n",
    "    def prediction(self,input_str):\n",
    "        try:\n",
    "            loaded_model = joblib.load(\"/Users/seethu-8363/Documents/Test/virenv1/model/sentiment_analysis_naive_bayes_model.pkl\")\n",
    "            res = loaded_model.predict([input_str])\n",
    "        except:\n",
    "            train_x , train_y , valid_x , valid_y = self.data_preproc()\n",
    "            self.load_model(train_x , train_y , valid_x , valid_y)\n",
    "            loaded_model = joblib.load(\"/Users/seethu-8363/Documents/Test/virenv1/model/sentiment_analysis_naive_bayes_model.pkl\")\n",
    "            res = loaded_model.predict([input_str])\n",
    "            \n",
    "        return res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}