BBC-CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import nltk \n",
    "import string\n",
    "import copy\n",
    "import random\n",
    "from os import listdir\n",
    "import pickle\n",
    "\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import string\n",
    "import re\n",
    "\n",
    "from nltk.stem.porter import *\n",
    "from nltk.corpus import stopwords\n",
    "from TurkishStemmer import TurkishStemmer\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import tree\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.linear_model import SGDClassifier,LinearRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "from pos_tagger import tag\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_dicts = pickle.load( open( \"./data/clean_data.p\", \"rb\" ) )\n",
    "\n",
    "article_text_dict_positive=all_dicts[\"article_text_dict_positive\"]\n",
    "iter1_BBC_text_dict_neg=all_dicts[\"iter1_BBC_text_dict_neg\"]\n",
    "iter2_BBC_text_dict_neg=all_dicts[\"iter2_BBC_text_dict_neg\"]\n",
    "iter1_CNN_neg_text=all_dicts[\"iter1_CNN_neg_text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1030\n",
      " 1031\n",
      " 948\n",
      " 1036\n"
     ]
    }
   ],
   "source": [
    "print(\"\",len(iter1_BBC_text_dict_neg))\n",
    "print(\"\",len(iter2_BBC_text_dict_neg))\n",
    "print(\"\",len(iter1_CNN_neg_text))\n",
    "print(\"\",len(article_text_dict_positive))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lowercase=' abcdefghijklmnoprstuvyzğöıüşç'\n",
    "stemmer = TurkishStemmer()\n",
    "\n",
    "# There are different tokenization functions\n",
    "\n",
    "# first one is tokenizes with nltk, lowers text \n",
    "# and applies stemming to each word\n",
    "def tokenize_stem(text):\n",
    "    \"\"\"Removes punctuation & excess whitespace, sets to lowercase,\n",
    "    and stems text. Returns a list of them\"\"\"\n",
    "    text=nltk.word_tokenize(text.lower().strip())\n",
    "    tokens = [stemmer.stem(t) for t in text]\n",
    "    return tokens\n",
    "\n",
    "\n",
    "# lowers text\n",
    "# generates part of speech tags  \n",
    "# returns \"word+tag\" \n",
    "# tokenization made with tagging library\n",
    "def pos_tokenize(text):\n",
    "    text=text.lower().strip()\n",
    "    tokens=[]\n",
    "    tags=tag(text)\n",
    "    for a_tag in tags:\n",
    "        tokens.append(a_tag[0]+\"+\"+a_tag[1])\n",
    "    return tokens\n",
    "\n",
    "# lowers text\n",
    "# splits text to it's characters\n",
    "def char_tokenize(text):\n",
    "    text = text.lower().strip()\n",
    "    tokens = [t for t in text]\n",
    "    return tokens\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# list of feature functions to be tested \n",
    "features={\"Pos_tags\":pos_tokenize,\"characters\":char_tokenize,\"tokenize\":nltk.word_tokenize,\"tokenize_stem\":tokenize_stem}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_experiment_w_features(X_train,X_test,y_train,y_test,feature,n_range,theclassifiers,vectorizer=None):\n",
    "    if vectorizer==None:\n",
    "        vectorizer = TfidfVectorizer(\n",
    "            tokenizer= features[feature],\n",
    "#             cleaning itself\n",
    "#             preprocessor=word_process_clean,\n",
    "            ngram_range=n_range,\n",
    "            use_idf=True,\n",
    "            min_df=0.003,\n",
    "            norm=None, )\n",
    "        \n",
    "    \n",
    "    X_train = vectorizer.fit_transform(X_train).toarray()\n",
    "    X_test = vectorizer.transform(X_test).toarray()\n",
    "\n",
    "    if type(theclassifiers)!=list:\n",
    "        theclassifiers=[theclassifiers]\n",
    "    for theclassifier in theclassifiers:\n",
    "#       initialise classifier\n",
    "        if theclassifier==LinearSVC:\n",
    "            clf= theclassifier(max_iter=3000)\n",
    "        else:\n",
    "            clf= theclassifier()\n",
    "#     create model with tranining data\n",
    "        model = clf.fit(X_train, y_train)\n",
    "#     predict test set\n",
    "        y_preds = model.predict(X_test)\n",
    "#     create the report\n",
    "        report = classification_report( y_test, y_preds )\n",
    "    \n",
    "#     find name of the classifier for printing\n",
    "        match=re.search(r\"\\.([A-z]*)'>\",str(theclassifier))\n",
    "        match=match.group(1)\n",
    "\n",
    "        result_text=\"\\033[1m Performance report of \\033[0m \\033[92m\" + feature +\"\\033[0m \"\n",
    "        count=0\n",
    "        for i in range(n_range[0],n_range[1]+1):\n",
    "            if count!=0:\n",
    "                result_text+=\" and \"\n",
    "            result_text=result_text+\"\\033[91m\"+str(i)+\"-gram\\033[0m\"\n",
    "            count+=1\n",
    "        result_text+= \" with \\033[94m\"+match+\"\\033[0m\"\n",
    "    #     print(\"Performance report of {} {}-gram\".format(feature,n_range[1]))\n",
    "        print (result_text)\n",
    "#         prnt_scores(report)\n",
    "        print(report)\n",
    "        print(accuracy_score( y_test, y_preds))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos examples 1036\n",
      "BBC 1031\n",
      "CNN 948\n",
      "pos examples 1036\n",
      "BBC 1036\n",
      "CNN 948\n",
      "train_pos 828 test_pos 208\n",
      "train_neg 824 test_neg 190\n",
      "X_train 1652 Y_train 1652 X_test 398 Y_test 398\n"
     ]
    }
   ],
   "source": [
    "pos_examp=[]\n",
    "neg_examp_train=[]\n",
    "neg_examp_test=[]\n",
    "\n",
    "print(\"pos examples\",len(article_text_dict_positive))\n",
    "print(\"BBC\",len(iter2_BBC_text_dict_neg))\n",
    "print(\"CNN\",len(iter1_CNN_neg_text))\n",
    "\n",
    "# some articles includes short text \n",
    "for artc in article_text_dict_positive.values():\n",
    "    pos_examp.append((artc,1))\n",
    "\n",
    "for artc in iter2_BBC_text_dict_neg.values():\n",
    "    neg_examp_train.append((artc,0))\n",
    "\n",
    "for artc in iter1_CNN_neg_text.values():\n",
    "    neg_examp_test.append((artc,0))\n",
    "\n",
    "print(\"pos examples\",len(pos_examp))\n",
    "print(\"BBC\",len(pos_examp))\n",
    "print(\"CNN\",len(neg_examp_test))\n",
    "\n",
    "# combine positive and negative samples then shuffle\n",
    "# XY=pos_examp+neg_examp_train\n",
    "random.seed(a=2)\n",
    "random.shuffle(pos_examp)\n",
    "random.shuffle(neg_examp_train)\n",
    "random.shuffle(neg_examp_test)\n",
    "\n",
    "percentage=0.8\n",
    "cut_point=int(len(pos_examp)*percentage)\n",
    "train_pos=pos_examp[:cut_point]\n",
    "test_pos=pos_examp[cut_point:]\n",
    "\n",
    "cut_point=int(len(neg_examp_train)*percentage)\n",
    "train_neg=neg_examp_train[:cut_point]\n",
    "cut_point=int(len(neg_examp_test)*percentage)\n",
    "test_neg=neg_examp_test[cut_point:]\n",
    "\n",
    "print(\"train_pos\",len(train_pos),\"test_pos\",len(test_pos))\n",
    "print(\"train_neg\",len(train_neg),\"test_neg\",len(test_neg))\n",
    "\n",
    "XY_train=train_pos+train_neg\n",
    "random.shuffle(XY_train)\n",
    "X_train=[k[0] for k in XY_train]\n",
    "Y_train=[k[1] for k in XY_train]\n",
    "\n",
    "XY_test=test_pos+test_neg\n",
    "random.shuffle(XY_test)\n",
    "X_test=[k[0] for k in XY_test]\n",
    "Y_test=[k[1] for k in XY_test]\n",
    "\n",
    "\n",
    "print(\"X_train\",len(X_train),\"Y_train\",len(Y_train),\"X_test\",len(X_test),\"Y_test\",len(Y_test))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "classifiers_list=[LinearSVC,MultinomialNB,RandomForestClassifier,\n",
    "                  AdaBoostClassifier]\n",
    "tryfeatures={\"tokenize\":nltk.word_tokenize,\"tokenize_stem\":tokenize_stem,\"Pos_tags\":pos_tokenize,\"characters\":char_tokenize}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.70      0.80       190\n",
      "           1       0.78      0.96      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.86      0.83      0.83       398\n",
      "weighted avg       0.86      0.84      0.83       398\n",
      "\n",
      "0.8366834170854272\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.69      0.80       190\n",
      "           1       0.77      0.97      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.87      0.83      0.83       398\n",
      "weighted avg       0.86      0.84      0.83       398\n",
      "\n",
      "0.8366834170854272\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.69      0.80       190\n",
      "           1       0.78      0.97      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.87      0.83      0.83       398\n",
      "weighted avg       0.86      0.84      0.84       398\n",
      "\n",
      "0.8391959798994975\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.42      0.55       190\n",
      "           1       0.63      0.91      0.75       208\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       398\n",
      "   macro avg       0.72      0.66      0.65       398\n",
      "weighted avg       0.72      0.68      0.65       398\n",
      "\n",
      "0.6758793969849246\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.42      0.56       190\n",
      "           1       0.64      0.92      0.75       208\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       398\n",
      "   macro avg       0.73      0.67      0.66       398\n",
      "weighted avg       0.73      0.68      0.66       398\n",
      "\n",
      "0.6834170854271356\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.22      0.33       190\n",
      "           1       0.57      0.93      0.70       208\n",
      "\n",
      "   micro avg       0.59      0.59      0.59       398\n",
      "   macro avg       0.66      0.57      0.52       398\n",
      "weighted avg       0.65      0.59      0.53       398\n",
      "\n",
      "0.5904522613065326\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.72      0.82       190\n",
      "           1       0.79      0.98      0.87       208\n",
      "\n",
      "   micro avg       0.85      0.85      0.85       398\n",
      "   macro avg       0.88      0.85      0.85       398\n",
      "weighted avg       0.87      0.85      0.85       398\n",
      "\n",
      "0.8517587939698492\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.73      0.83       190\n",
      "           1       0.80      0.98      0.88       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.88      0.85      0.86       398\n",
      "weighted avg       0.88      0.86      0.86       398\n",
      "\n",
      "0.8592964824120602\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.72      0.83       190\n",
      "           1       0.79      0.98      0.88       208\n",
      "\n",
      "   micro avg       0.85      0.85      0.85       398\n",
      "   macro avg       0.88      0.85      0.85       398\n",
      "weighted avg       0.87      0.85      0.85       398\n",
      "\n",
      "0.8542713567839196\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.55      0.67       190\n",
      "           1       0.69      0.92      0.79       208\n",
      "\n",
      "   micro avg       0.74      0.74      0.74       398\n",
      "   macro avg       0.78      0.74      0.73       398\n",
      "weighted avg       0.77      0.74      0.73       398\n",
      "\n",
      "0.7437185929648241\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.55      0.67       190\n",
      "           1       0.69      0.92      0.79       208\n",
      "\n",
      "   micro avg       0.74      0.74      0.74       398\n",
      "   macro avg       0.77      0.73      0.73       398\n",
      "weighted avg       0.77      0.74      0.73       398\n",
      "\n",
      "0.7412060301507538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.28      0.41       190\n",
      "           1       0.58      0.89      0.70       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.64      0.59      0.55       398\n",
      "weighted avg       0.64      0.60      0.56       398\n",
      "\n",
      "0.6030150753768844\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.70      0.80       190\n",
      "           1       0.78      0.96      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.86      0.83      0.83       398\n",
      "weighted avg       0.86      0.84      0.83       398\n",
      "\n",
      "0.8366834170854272\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.69      0.80       190\n",
      "           1       0.77      0.97      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.87      0.83      0.83       398\n",
      "weighted avg       0.86      0.84      0.83       398\n",
      "\n",
      "0.8366834170854272\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.69      0.80       190\n",
      "           1       0.78      0.97      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.87      0.83      0.83       398\n",
      "weighted avg       0.86      0.84      0.84       398\n",
      "\n",
      "0.8391959798994975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.42      0.55       190\n",
      "           1       0.63      0.91      0.75       208\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       398\n",
      "   macro avg       0.72      0.66      0.65       398\n",
      "weighted avg       0.72      0.68      0.65       398\n",
      "\n",
      "0.6758793969849246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.42      0.56       190\n",
      "           1       0.64      0.92      0.75       208\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       398\n",
      "   macro avg       0.73      0.67      0.66       398\n",
      "weighted avg       0.73      0.68      0.66       398\n",
      "\n",
      "0.6834170854271356\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.22      0.33       190\n",
      "           1       0.57      0.93      0.70       208\n",
      "\n",
      "   micro avg       0.59      0.59      0.59       398\n",
      "   macro avg       0.66      0.57      0.52       398\n",
      "weighted avg       0.65      0.59      0.53       398\n",
      "\n",
      "0.5904522613065326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.55      0.67       190\n",
      "           1       0.69      0.92      0.79       208\n",
      "\n",
      "   micro avg       0.74      0.74      0.74       398\n",
      "   macro avg       0.77      0.73      0.73       398\n",
      "weighted avg       0.77      0.74      0.73       398\n",
      "\n",
      "0.7412060301507538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.62      0.74       190\n",
      "           1       0.73      0.96      0.83       208\n",
      "\n",
      "   micro avg       0.80      0.80      0.80       398\n",
      "   macro avg       0.83      0.79      0.79       398\n",
      "weighted avg       0.83      0.80      0.79       398\n",
      "\n",
      "0.7964824120603015\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.64      0.77       190\n",
      "           1       0.75      0.98      0.85       208\n",
      "\n",
      "   micro avg       0.81      0.81      0.81       398\n",
      "   macro avg       0.85      0.81      0.81       398\n",
      "weighted avg       0.85      0.81      0.81       398\n",
      "\n",
      "0.8140703517587939\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/svm/base.py:931: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.63      0.75       190\n",
      "           1       0.74      0.95      0.83       208\n",
      "\n",
      "   micro avg       0.80      0.80      0.80       398\n",
      "   macro avg       0.83      0.79      0.79       398\n",
      "weighted avg       0.83      0.80      0.79       398\n",
      "\n",
      "0.7964824120603015\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.64      0.77       190\n",
      "           1       0.75      0.97      0.85       208\n",
      "\n",
      "   micro avg       0.81      0.81      0.81       398\n",
      "   macro avg       0.85      0.81      0.81       398\n",
      "weighted avg       0.85      0.81      0.81       398\n",
      "\n",
      "0.8140703517587939\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mLinearSVC\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.66      0.79       190\n",
      "           1       0.76      0.98      0.86       208\n",
      "\n",
      "   micro avg       0.83      0.83      0.83       398\n",
      "   macro avg       0.87      0.82      0.82       398\n",
      "weighted avg       0.86      0.83      0.82       398\n",
      "\n",
      "0.8291457286432161\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.83      0.88       190\n",
      "           1       0.86      0.96      0.91       208\n",
      "\n",
      "   micro avg       0.90      0.90      0.90       398\n",
      "   macro avg       0.90      0.89      0.90       398\n",
      "weighted avg       0.90      0.90      0.90       398\n",
      "\n",
      "0.8969849246231156\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.83      0.88       190\n",
      "           1       0.86      0.96      0.90       208\n",
      "\n",
      "   micro avg       0.89      0.89      0.89       398\n",
      "   macro avg       0.90      0.89      0.89       398\n",
      "weighted avg       0.90      0.89      0.89       398\n",
      "\n",
      "0.8944723618090452\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.74      0.81       190\n",
      "           1       0.79      0.93      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.85      0.83      0.83       398\n",
      "weighted avg       0.85      0.84      0.83       398\n",
      "\n",
      "0.8366834170854272\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.73      0.81       190\n",
      "           1       0.79      0.94      0.86       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.85      0.83      0.84       398\n",
      "weighted avg       0.85      0.84      0.84       398\n",
      "\n",
      "0.8391959798994975\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.41      0.53       190\n",
      "           1       0.62      0.89      0.73       208\n",
      "\n",
      "   micro avg       0.66      0.66      0.66       398\n",
      "   macro avg       0.70      0.65      0.63       398\n",
      "weighted avg       0.69      0.66      0.64       398\n",
      "\n",
      "0.6582914572864321\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.86      0.90       190\n",
      "           1       0.88      0.95      0.91       208\n",
      "\n",
      "   micro avg       0.91      0.91      0.91       398\n",
      "   macro avg       0.91      0.90      0.91       398\n",
      "weighted avg       0.91      0.91      0.91       398\n",
      "\n",
      "0.907035175879397\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.85      0.89       190\n",
      "           1       0.87      0.95      0.91       208\n",
      "\n",
      "   micro avg       0.90      0.90      0.90       398\n",
      "   macro avg       0.91      0.90      0.90       398\n",
      "weighted avg       0.91      0.90      0.90       398\n",
      "\n",
      "0.9020100502512562\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.85      0.90       190\n",
      "           1       0.87      0.96      0.92       208\n",
      "\n",
      "   micro avg       0.91      0.91      0.91       398\n",
      "   macro avg       0.91      0.90      0.91       398\n",
      "weighted avg       0.91      0.91      0.91       398\n",
      "\n",
      "0.907035175879397\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.76      0.81       190\n",
      "           1       0.80      0.90      0.85       208\n",
      "\n",
      "   micro avg       0.83      0.83      0.83       398\n",
      "   macro avg       0.84      0.83      0.83       398\n",
      "weighted avg       0.84      0.83      0.83       398\n",
      "\n",
      "0.8341708542713567\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.76      0.81       190\n",
      "           1       0.80      0.90      0.85       208\n",
      "\n",
      "   micro avg       0.83      0.83      0.83       398\n",
      "   macro avg       0.84      0.83      0.83       398\n",
      "weighted avg       0.84      0.83      0.83       398\n",
      "\n",
      "0.8341708542713567\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.35      0.48       190\n",
      "           1       0.60      0.90      0.72       208\n",
      "\n",
      "   micro avg       0.64      0.64      0.64       398\n",
      "   macro avg       0.68      0.62      0.60       398\n",
      "weighted avg       0.68      0.64      0.60       398\n",
      "\n",
      "0.635678391959799\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.77      0.75       190\n",
      "           1       0.78      0.75      0.76       208\n",
      "\n",
      "   micro avg       0.76      0.76      0.76       398\n",
      "   macro avg       0.76      0.76      0.76       398\n",
      "weighted avg       0.76      0.76      0.76       398\n",
      "\n",
      "0.7587939698492462\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.87      0.86       190\n",
      "           1       0.88      0.85      0.87       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.86      0.86      0.86       398\n",
      "weighted avg       0.86      0.86      0.86       398\n",
      "\n",
      "0.8618090452261307\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.88      0.90       190\n",
      "           1       0.90      0.92      0.91       208\n",
      "\n",
      "   micro avg       0.90      0.90      0.90       398\n",
      "   macro avg       0.91      0.90      0.90       398\n",
      "weighted avg       0.90      0.90      0.90       398\n",
      "\n",
      "0.9045226130653267\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.87      0.86       190\n",
      "           1       0.88      0.86      0.87       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.86      0.86      0.86       398\n",
      "weighted avg       0.86      0.86      0.86       398\n",
      "\n",
      "0.8618090452261307\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.88      0.90       190\n",
      "           1       0.90      0.92      0.91       208\n",
      "\n",
      "   micro avg       0.90      0.90      0.90       398\n",
      "   macro avg       0.91      0.90      0.90       398\n",
      "weighted avg       0.90      0.90      0.90       398\n",
      "\n",
      "0.9045226130653267\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMultinomialNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.87      0.90       190\n",
      "           1       0.89      0.94      0.91       208\n",
      "\n",
      "   micro avg       0.90      0.90      0.90       398\n",
      "   macro avg       0.91      0.90      0.90       398\n",
      "weighted avg       0.91      0.90      0.90       398\n",
      "\n",
      "0.9045226130653267\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.85      0.85       190\n",
      "           1       0.87      0.87      0.87       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.86      0.86      0.86       398\n",
      "weighted avg       0.86      0.86      0.86       398\n",
      "\n",
      "0.8618090452261307\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.77      0.80       190\n",
      "           1       0.80      0.85      0.83       208\n",
      "\n",
      "   micro avg       0.81      0.81      0.81       398\n",
      "   macro avg       0.82      0.81      0.81       398\n",
      "weighted avg       0.81      0.81      0.81       398\n",
      "\n",
      "0.8140703517587939\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.82      0.83       190\n",
      "           1       0.84      0.86      0.85       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.84      0.84      0.84       398\n",
      "weighted avg       0.84      0.84      0.84       398\n",
      "\n",
      "0.8417085427135679\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.49      0.57       190\n",
      "           1       0.63      0.80      0.71       208\n",
      "\n",
      "   micro avg       0.65      0.65      0.65       398\n",
      "   macro avg       0.66      0.65      0.64       398\n",
      "weighted avg       0.66      0.65      0.64       398\n",
      "\n",
      "0.6532663316582915\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.47      0.57       190\n",
      "           1       0.64      0.85      0.73       208\n",
      "\n",
      "   micro avg       0.67      0.67      0.67       398\n",
      "   macro avg       0.69      0.66      0.65       398\n",
      "weighted avg       0.69      0.67      0.65       398\n",
      "\n",
      "0.6683417085427136\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.25      0.38       190\n",
      "           1       0.58      0.94      0.71       208\n",
      "\n",
      "   micro avg       0.61      0.61      0.61       398\n",
      "   macro avg       0.68      0.59      0.55       398\n",
      "weighted avg       0.68      0.61      0.55       398\n",
      "\n",
      "0.6080402010050251\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.89      0.86       190\n",
      "           1       0.89      0.83      0.86       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.86      0.86      0.86       398\n",
      "weighted avg       0.86      0.86      0.86       398\n",
      "\n",
      "0.8567839195979899\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.88      0.85       190\n",
      "           1       0.89      0.83      0.86       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.86      0.86      0.86       398\n",
      "weighted avg       0.86      0.86      0.86       398\n",
      "\n",
      "0.8567839195979899\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.88      0.89       190\n",
      "           1       0.90      0.90      0.90       208\n",
      "\n",
      "   micro avg       0.89      0.89      0.89       398\n",
      "   macro avg       0.89      0.89      0.89       398\n",
      "weighted avg       0.89      0.89      0.89       398\n",
      "\n",
      "0.8944723618090452\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.57      0.64       190\n",
      "           1       0.67      0.82      0.74       208\n",
      "\n",
      "   micro avg       0.70      0.70      0.70       398\n",
      "   macro avg       0.71      0.69      0.69       398\n",
      "weighted avg       0.71      0.70      0.69       398\n",
      "\n",
      "0.6984924623115578\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.51      0.60       190\n",
      "           1       0.65      0.83      0.73       208\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       398\n",
      "   macro avg       0.69      0.67      0.66       398\n",
      "weighted avg       0.69      0.68      0.67       398\n",
      "\n",
      "0.6758793969849246\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.29      0.42       190\n",
      "           1       0.58      0.90      0.71       208\n",
      "\n",
      "   micro avg       0.61      0.61      0.61       398\n",
      "   macro avg       0.65      0.60      0.56       398\n",
      "weighted avg       0.65      0.61      0.57       398\n",
      "\n",
      "0.6105527638190955\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.82      0.81       190\n",
      "           1       0.83      0.82      0.82       208\n",
      "\n",
      "   micro avg       0.82      0.82      0.82       398\n",
      "   macro avg       0.82      0.82      0.82       398\n",
      "weighted avg       0.82      0.82      0.82       398\n",
      "\n",
      "0.8165829145728644\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.85      0.84       190\n",
      "           1       0.86      0.85      0.86       208\n",
      "\n",
      "   micro avg       0.85      0.85      0.85       398\n",
      "   macro avg       0.85      0.85      0.85       398\n",
      "weighted avg       0.85      0.85      0.85       398\n",
      "\n",
      "0.8492462311557789\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.90      0.86       190\n",
      "           1       0.90      0.82      0.86       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.86      0.86      0.86       398\n",
      "weighted avg       0.86      0.86      0.86       398\n",
      "\n",
      "0.8567839195979899\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.65      0.47      0.55       190\n",
      "           1       0.61      0.77      0.69       208\n",
      "\n",
      "   micro avg       0.63      0.63      0.63       398\n",
      "   macro avg       0.63      0.62      0.62       398\n",
      "weighted avg       0.63      0.63      0.62       398\n",
      "\n",
      "0.628140703517588\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.47      0.57       190\n",
      "           1       0.63      0.82      0.71       208\n",
      "\n",
      "   micro avg       0.66      0.66      0.66       398\n",
      "   macro avg       0.67      0.65      0.64       398\n",
      "weighted avg       0.67      0.66      0.64       398\n",
      "\n",
      "0.6557788944723618\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.26      0.38       190\n",
      "           1       0.57      0.91      0.70       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.65      0.58      0.54       398\n",
      "weighted avg       0.64      0.60      0.55       398\n",
      "\n",
      "0.5979899497487438\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.66      0.68      0.67       190\n",
      "           1       0.70      0.69      0.69       208\n",
      "\n",
      "   micro avg       0.68      0.68      0.68       398\n",
      "   macro avg       0.68      0.68      0.68       398\n",
      "weighted avg       0.68      0.68      0.68       398\n",
      "\n",
      "0.6834170854271356\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.77      0.78       190\n",
      "           1       0.80      0.82      0.81       208\n",
      "\n",
      "   micro avg       0.80      0.80      0.80       398\n",
      "   macro avg       0.80      0.80      0.80       398\n",
      "weighted avg       0.80      0.80      0.80       398\n",
      "\n",
      "0.7964824120603015\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.76      0.79       190\n",
      "           1       0.79      0.84      0.81       208\n",
      "\n",
      "   micro avg       0.80      0.80      0.80       398\n",
      "   macro avg       0.80      0.80      0.80       398\n",
      "weighted avg       0.80      0.80      0.80       398\n",
      "\n",
      "0.8015075376884422\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.76      0.80       190\n",
      "           1       0.80      0.88      0.83       208\n",
      "\n",
      "   micro avg       0.82      0.82      0.82       398\n",
      "   macro avg       0.82      0.82      0.82       398\n",
      "weighted avg       0.82      0.82      0.82       398\n",
      "\n",
      "0.8190954773869347\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.81      0.83       190\n",
      "           1       0.83      0.87      0.85       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.84      0.84      0.84       398\n",
      "weighted avg       0.84      0.84      0.84       398\n",
      "\n",
      "0.8391959798994975\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mRandomForestClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.73      0.77       190\n",
      "           1       0.77      0.84      0.81       208\n",
      "\n",
      "   micro avg       0.79      0.79      0.79       398\n",
      "   macro avg       0.79      0.79      0.79       398\n",
      "weighted avg       0.79      0.79      0.79       398\n",
      "\n",
      "0.7889447236180904\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.54      0.64       190\n",
      "           1       0.67      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.72      0.71      0.70       398\n",
      "\n",
      "0.7085427135678392\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.54      0.64       190\n",
      "           1       0.67      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.72      0.71      0.70       398\n",
      "\n",
      "0.7085427135678392\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.54      0.64       190\n",
      "           1       0.67      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.72      0.71      0.70       398\n",
      "\n",
      "0.7085427135678392\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.38      0.51       190\n",
      "           1       0.61      0.88      0.72       208\n",
      "\n",
      "   micro avg       0.64      0.64      0.64       398\n",
      "   macro avg       0.68      0.63      0.61       398\n",
      "weighted avg       0.67      0.64      0.62       398\n",
      "\n",
      "0.6432160804020101\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.38      0.51       190\n",
      "           1       0.61      0.88      0.72       208\n",
      "\n",
      "   micro avg       0.64      0.64      0.64       398\n",
      "   macro avg       0.68      0.63      0.61       398\n",
      "weighted avg       0.67      0.64      0.62       398\n",
      "\n",
      "0.6432160804020101\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.08      0.15       190\n",
      "           1       0.54      0.99      0.70       208\n",
      "\n",
      "   micro avg       0.56      0.56      0.56       398\n",
      "   macro avg       0.72      0.54      0.43       398\n",
      "weighted avg       0.71      0.56      0.44       398\n",
      "\n",
      "0.5577889447236181\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.61      0.70       190\n",
      "           1       0.71      0.88      0.79       208\n",
      "\n",
      "   micro avg       0.75      0.75      0.75       398\n",
      "   macro avg       0.77      0.74      0.74       398\n",
      "weighted avg       0.77      0.75      0.75       398\n",
      "\n",
      "0.7512562814070352\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.63      0.71       190\n",
      "           1       0.72      0.87      0.79       208\n",
      "\n",
      "   micro avg       0.76      0.76      0.76       398\n",
      "   macro avg       0.77      0.75      0.75       398\n",
      "weighted avg       0.77      0.76      0.75       398\n",
      "\n",
      "0.7562814070351759\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.63      0.71       190\n",
      "           1       0.72      0.87      0.79       208\n",
      "\n",
      "   micro avg       0.76      0.76      0.76       398\n",
      "   macro avg       0.77      0.75      0.75       398\n",
      "weighted avg       0.77      0.76      0.75       398\n",
      "\n",
      "0.7562814070351759\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.49      0.61       190\n",
      "           1       0.66      0.90      0.76       208\n",
      "\n",
      "   micro avg       0.70      0.70      0.70       398\n",
      "   macro avg       0.74      0.69      0.69       398\n",
      "weighted avg       0.73      0.70      0.69       398\n",
      "\n",
      "0.7035175879396985\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.49      0.61       190\n",
      "           1       0.66      0.90      0.76       208\n",
      "\n",
      "   micro avg       0.70      0.70      0.70       398\n",
      "   macro avg       0.74      0.69      0.69       398\n",
      "weighted avg       0.73      0.70      0.69       398\n",
      "\n",
      "0.7035175879396985\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.12      0.21       190\n",
      "           1       0.55      0.99      0.71       208\n",
      "\n",
      "   micro avg       0.57      0.57      0.57       398\n",
      "   macro avg       0.72      0.55      0.46       398\n",
      "weighted avg       0.71      0.57      0.47       398\n",
      "\n",
      "0.5728643216080402\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.54      0.64       190\n",
      "           1       0.67      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.72      0.71      0.70       398\n",
      "\n",
      "0.7085427135678392\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.54      0.64       190\n",
      "           1       0.67      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.72      0.71      0.70       398\n",
      "\n",
      "0.7085427135678392\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.54      0.64       190\n",
      "           1       0.67      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.72      0.71      0.70       398\n",
      "\n",
      "0.7085427135678392\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.38      0.51       190\n",
      "           1       0.61      0.88      0.72       208\n",
      "\n",
      "   micro avg       0.64      0.64      0.64       398\n",
      "   macro avg       0.68      0.63      0.61       398\n",
      "weighted avg       0.67      0.64      0.62       398\n",
      "\n",
      "0.6432160804020101\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.38      0.51       190\n",
      "           1       0.61      0.88      0.72       208\n",
      "\n",
      "   micro avg       0.64      0.64      0.64       398\n",
      "   macro avg       0.68      0.63      0.61       398\n",
      "weighted avg       0.67      0.64      0.62       398\n",
      "\n",
      "0.6432160804020101\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.08      0.15       190\n",
      "           1       0.54      0.99      0.70       208\n",
      "\n",
      "   micro avg       0.56      0.56      0.56       398\n",
      "   macro avg       0.72      0.54      0.43       398\n",
      "weighted avg       0.71      0.56      0.44       398\n",
      "\n",
      "0.5577889447236181\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.60      0.65       190\n",
      "           1       0.68      0.77      0.72       208\n",
      "\n",
      "   micro avg       0.69      0.69      0.69       398\n",
      "   macro avg       0.69      0.68      0.68       398\n",
      "weighted avg       0.69      0.69      0.69       398\n",
      "\n",
      "0.6884422110552764\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.67      0.77       190\n",
      "           1       0.76      0.93      0.83       208\n",
      "\n",
      "   micro avg       0.81      0.81      0.81       398\n",
      "   macro avg       0.83      0.80      0.80       398\n",
      "weighted avg       0.82      0.81      0.80       398\n",
      "\n",
      "0.8065326633165829\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m and \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.57      0.70       190\n",
      "           1       0.71      0.94      0.81       208\n",
      "\n",
      "   micro avg       0.76      0.76      0.76       398\n",
      "   macro avg       0.80      0.76      0.75       398\n",
      "weighted avg       0.80      0.76      0.75       398\n",
      "\n",
      "0.7638190954773869\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.62      0.73       190\n",
      "           1       0.73      0.93      0.81       208\n",
      "\n",
      "   micro avg       0.78      0.78      0.78       398\n",
      "   macro avg       0.81      0.77      0.77       398\n",
      "weighted avg       0.80      0.78      0.77       398\n",
      "\n",
      "0.7788944723618091\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m and \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.57      0.70       190\n",
      "           1       0.71      0.94      0.81       208\n",
      "\n",
      "   micro avg       0.76      0.76      0.76       398\n",
      "   macro avg       0.80      0.76      0.75       398\n",
      "weighted avg       0.80      0.76      0.75       398\n",
      "\n",
      "0.7638190954773869\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mAdaBoostClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.45      0.60       190\n",
      "           1       0.66      0.96      0.78       208\n",
      "\n",
      "   micro avg       0.72      0.72      0.72       398\n",
      "   macro avg       0.78      0.70      0.69       398\n",
      "weighted avg       0.78      0.72      0.69       398\n",
      "\n",
      "0.7160804020100503\n"
     ]
    }
   ],
   "source": [
    "for clasifier in classifiers_list:\n",
    "    for feature in tryfeatures:\n",
    "        for i in range(1,4):\n",
    "            for k in range(i,4):\n",
    "                run_experiment_w_features(X_train[:],X_test[:],Y_train[:],Y_test[:],feature,(i,k),clasifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.61      0.67       190\n",
      "           1       0.69      0.81      0.75       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.72      0.71      0.71       398\n",
      "weighted avg       0.72      0.71      0.71       398\n",
      "\n",
      "0.7135678391959799\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.65      0.49      0.56       190\n",
      "           1       0.62      0.75      0.68       208\n",
      "\n",
      "   micro avg       0.63      0.63      0.63       398\n",
      "   macro avg       0.63      0.62      0.62       398\n",
      "weighted avg       0.63      0.63      0.62       398\n",
      "\n",
      "0.628140703517588\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.27      0.39       190\n",
      "           1       0.57      0.90      0.70       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.64      0.58      0.54       398\n",
      "weighted avg       0.64      0.60      0.55       398\n",
      "\n",
      "0.5979899497487438\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.63      0.70       190\n",
      "           1       0.72      0.85      0.78       208\n",
      "\n",
      "   micro avg       0.75      0.75      0.75       398\n",
      "   macro avg       0.76      0.74      0.74       398\n",
      "weighted avg       0.75      0.75      0.74       398\n",
      "\n",
      "0.7462311557788944\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.51      0.59       190\n",
      "           1       0.64      0.81      0.72       208\n",
      "\n",
      "   micro avg       0.67      0.67      0.67       398\n",
      "   macro avg       0.68      0.66      0.65       398\n",
      "weighted avg       0.67      0.67      0.66       398\n",
      "\n",
      "0.6658291457286433\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.33      0.46       190\n",
      "           1       0.59      0.89      0.71       208\n",
      "\n",
      "   micro avg       0.62      0.62      0.62       398\n",
      "   macro avg       0.66      0.61      0.58       398\n",
      "weighted avg       0.66      0.62      0.59       398\n",
      "\n",
      "0.6231155778894473\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.63      0.68       190\n",
      "           1       0.70      0.80      0.75       208\n",
      "\n",
      "   micro avg       0.72      0.72      0.72       398\n",
      "   macro avg       0.72      0.71      0.71       398\n",
      "weighted avg       0.72      0.72      0.71       398\n",
      "\n",
      "0.7160804020100503\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.47      0.54       190\n",
      "           1       0.61      0.75      0.68       208\n",
      "\n",
      "   micro avg       0.62      0.62      0.62       398\n",
      "   macro avg       0.62      0.61      0.61       398\n",
      "weighted avg       0.62      0.62      0.61       398\n",
      "\n",
      "0.6206030150753769\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.27      0.39       190\n",
      "           1       0.57      0.90      0.70       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.64      0.58      0.54       398\n",
      "weighted avg       0.64      0.60      0.55       398\n",
      "\n",
      "0.5979899497487438\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.66      0.57      0.62       190\n",
      "           1       0.65      0.74      0.69       208\n",
      "\n",
      "   micro avg       0.66      0.66      0.66       398\n",
      "   macro avg       0.66      0.65      0.65       398\n",
      "weighted avg       0.66      0.66      0.66       398\n",
      "\n",
      "0.6582914572864321\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.67      0.73       190\n",
      "           1       0.74      0.84      0.78       208\n",
      "\n",
      "   micro avg       0.76      0.76      0.76       398\n",
      "   macro avg       0.76      0.75      0.76       398\n",
      "weighted avg       0.76      0.76      0.76       398\n",
      "\n",
      "0.7587939698492462\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mDecisionTreeClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.49      0.61       190\n",
      "           1       0.66      0.88      0.75       208\n",
      "\n",
      "   micro avg       0.70      0.70      0.70       398\n",
      "   macro avg       0.72      0.69      0.68       398\n",
      "weighted avg       0.72      0.70      0.68       398\n",
      "\n",
      "0.6959798994974874\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.66      0.78       190\n",
      "           1       0.76      0.96      0.85       208\n",
      "\n",
      "   micro avg       0.82      0.82      0.82       398\n",
      "   macro avg       0.85      0.81      0.81       398\n",
      "weighted avg       0.84      0.82      0.81       398\n",
      "\n",
      "0.8190954773869347\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.55      0.64       190\n",
      "           1       0.68      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.73      0.71      0.70       398\n",
      "\n",
      "0.7110552763819096\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.21      0.33       190\n",
      "           1       0.57      0.96      0.72       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.70      0.58      0.52       398\n",
      "weighted avg       0.69      0.60      0.53       398\n",
      "\n",
      "0.6005025125628141\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.68      0.79       190\n",
      "           1       0.77      0.96      0.85       208\n",
      "\n",
      "   micro avg       0.83      0.83      0.83       398\n",
      "   macro avg       0.86      0.82      0.82       398\n",
      "weighted avg       0.85      0.83      0.83       398\n",
      "\n",
      "0.8291457286432161\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.64      0.73       190\n",
      "           1       0.73      0.90      0.81       208\n",
      "\n",
      "   micro avg       0.78      0.78      0.78       398\n",
      "   macro avg       0.79      0.77      0.77       398\n",
      "weighted avg       0.79      0.78      0.77       398\n",
      "\n",
      "0.7763819095477387\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.20      0.32       190\n",
      "           1       0.56      0.95      0.71       208\n",
      "\n",
      "   micro avg       0.59      0.59      0.59       398\n",
      "   macro avg       0.67      0.57      0.51       398\n",
      "weighted avg       0.67      0.59      0.52       398\n",
      "\n",
      "0.5904522613065326\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.66      0.78       190\n",
      "           1       0.76      0.96      0.85       208\n",
      "\n",
      "   micro avg       0.82      0.82      0.82       398\n",
      "   macro avg       0.85      0.81      0.81       398\n",
      "weighted avg       0.84      0.82      0.81       398\n",
      "\n",
      "0.8190954773869347\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.55      0.64       190\n",
      "           1       0.68      0.86      0.76       208\n",
      "\n",
      "   micro avg       0.71      0.71      0.71       398\n",
      "   macro avg       0.73      0.70      0.70       398\n",
      "weighted avg       0.73      0.71      0.70       398\n",
      "\n",
      "0.7110552763819096\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.21      0.33       190\n",
      "           1       0.57      0.96      0.72       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.70      0.58      0.52       398\n",
      "weighted avg       0.69      0.60      0.53       398\n",
      "\n",
      "0.6005025125628141\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.70      0.71       190\n",
      "           1       0.73      0.75      0.74       208\n",
      "\n",
      "   micro avg       0.73      0.73      0.73       398\n",
      "   macro avg       0.73      0.72      0.73       398\n",
      "weighted avg       0.73      0.73      0.73       398\n",
      "\n",
      "0.7261306532663316\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.94      0.85       190\n",
      "           1       0.93      0.75      0.83       208\n",
      "\n",
      "   micro avg       0.84      0.84      0.84       398\n",
      "   macro avg       0.85      0.85      0.84       398\n",
      "weighted avg       0.86      0.84      0.84       398\n",
      "\n",
      "0.8417085427135679\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mGaussianNB\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.68      0.76       190\n",
      "           1       0.76      0.90      0.82       208\n",
      "\n",
      "   micro avg       0.80      0.80      0.80       398\n",
      "   macro avg       0.81      0.79      0.79       398\n",
      "weighted avg       0.81      0.80      0.80       398\n",
      "\n",
      "0.7989949748743719\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.72      0.83       190\n",
      "           1       0.80      0.99      0.88       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.89      0.86      0.86       398\n",
      "weighted avg       0.89      0.86      0.86       398\n",
      "\n",
      "0.8618090452261307\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.63      0.73       190\n",
      "           1       0.73      0.91      0.81       208\n",
      "\n",
      "   micro avg       0.78      0.78      0.78       398\n",
      "   macro avg       0.80      0.77      0.77       398\n",
      "weighted avg       0.80      0.78      0.77       398\n",
      "\n",
      "0.7788944723618091\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.72      0.26      0.38       190\n",
      "           1       0.57      0.91      0.70       208\n",
      "\n",
      "   micro avg       0.60      0.60      0.60       398\n",
      "   macro avg       0.65      0.58      0.54       398\n",
      "weighted avg       0.64      0.60      0.55       398\n",
      "\n",
      "0.5979899497487438\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.77      0.86       190\n",
      "           1       0.83      0.99      0.90       208\n",
      "\n",
      "   micro avg       0.88      0.88      0.88       398\n",
      "   macro avg       0.90      0.88      0.88       398\n",
      "weighted avg       0.90      0.88      0.88       398\n",
      "\n",
      "0.8844221105527639\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.64      0.76       190\n",
      "           1       0.74      0.96      0.84       208\n",
      "\n",
      "   micro avg       0.81      0.81      0.81       398\n",
      "   macro avg       0.84      0.80      0.80       398\n",
      "weighted avg       0.84      0.81      0.80       398\n",
      "\n",
      "0.8065326633165829\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mtokenize_stem\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.33      0.46       190\n",
      "           1       0.59      0.90      0.72       208\n",
      "\n",
      "   micro avg       0.63      0.63      0.63       398\n",
      "   macro avg       0.68      0.62      0.59       398\n",
      "weighted avg       0.67      0.63      0.59       398\n",
      "\n",
      "0.628140703517588\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.77      0.86       190\n",
      "           1       0.83      0.98      0.90       208\n",
      "\n",
      "   micro avg       0.88      0.88      0.88       398\n",
      "   macro avg       0.90      0.88      0.88       398\n",
      "weighted avg       0.90      0.88      0.88       398\n",
      "\n",
      "0.8819095477386935\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.59      0.71       190\n",
      "           1       0.72      0.93      0.81       208\n",
      "\n",
      "   micro avg       0.77      0.77      0.77       398\n",
      "   macro avg       0.80      0.76      0.76       398\n",
      "weighted avg       0.80      0.77      0.76       398\n",
      "\n",
      "0.7713567839195979\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mPos_tags\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.75      0.26      0.39       190\n",
      "           1       0.58      0.92      0.71       208\n",
      "\n",
      "   micro avg       0.61      0.61      0.61       398\n",
      "   macro avg       0.66      0.59      0.55       398\n",
      "weighted avg       0.66      0.61      0.56       398\n",
      "\n",
      "0.6055276381909548\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m1-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.76      0.78       190\n",
      "           1       0.79      0.84      0.81       208\n",
      "\n",
      "   micro avg       0.80      0.80      0.80       398\n",
      "   macro avg       0.80      0.80      0.80       398\n",
      "weighted avg       0.80      0.80      0.80       398\n",
      "\n",
      "0.7989949748743719\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m2-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.69      0.79       190\n",
      "           1       0.77      0.94      0.85       208\n",
      "\n",
      "   micro avg       0.82      0.82      0.82       398\n",
      "   macro avg       0.84      0.82      0.82       398\n",
      "weighted avg       0.84      0.82      0.82       398\n",
      "\n",
      "0.821608040201005\n",
      "\u001b[1m Performance report of \u001b[0m \u001b[92mcharacters\u001b[0m \u001b[91m3-gram\u001b[0m with \u001b[94mMLPClassifier\u001b[0m\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.75      0.84       190\n",
      "           1       0.81      0.97      0.88       208\n",
      "\n",
      "   micro avg       0.86      0.86      0.86       398\n",
      "   macro avg       0.88      0.86      0.86       398\n",
      "weighted avg       0.88      0.86      0.86       398\n",
      "\n",
      "0.864321608040201\n"
     ]
    }
   ],
   "source": [
    "slow_classifiers_list=[DecisionTreeClassifier,GaussianNB,MLPClassifier]\n",
    "for clasifier in slow_classifiers_list:\n",
    "    for feature in tryfeatures:\n",
    "        for i in range(1,4):\n",
    "            run_experiment_w_features(X_train[:],X_test[:],Y_train[:],Y_test[:],feature,(i,i),clasifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}