data_modeling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ef77e17-aab2-426c-b246-5f4a60cd271a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "from sklearn import metrics\n",
    "\n",
    "import scripts.blosc_interface as bi\n",
    "import scripts.corpus_split as cs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aef6301",
   "metadata": {},
   "source": [
    "https://stackoverflow.com/questions/26826002/adding-words-to-stop-words-list-in-tfidfvectorizer-in-sklearn\n",
    "https://stackoverflow.com/questions/50756085/how-to-print-the-progress-of-a-list-comprehension-in-python\n",
    "https://stackoverflow.com/questions/57983431/whats-the-most-space-efficient-way-to-compress-serialized-python-data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "968d3a9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_corpus = bi.blosc_read(\"./data/tokenized_corpus.dat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bb7a821a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Splitting the Data into Train and Test\n"
     ]
    }
   ],
   "source": [
    "x_train, x_test, y_train, y_test = cs.corpus_split(full_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "51678ffa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_vect__ngram_range</th>\n",
       "      <th>param_vect__max_features</th>\n",
       "      <th>param_svd__n_components</th>\n",
       "      <th>param_log_clf__C</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>64.123026</td>\n",
       "      <td>3.714061</td>\n",
       "      <td>6.076129</td>\n",
       "      <td>1.974486</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>30</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.848319</td>\n",
       "      <td>0.839127</td>\n",
       "      <td>0.827636</td>\n",
       "      <td>0.837690</td>\n",
       "      <td>0.850043</td>\n",
       "      <td>0.840563</td>\n",
       "      <td>0.008093</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>58.169902</td>\n",
       "      <td>0.920133</td>\n",
       "      <td>4.012292</td>\n",
       "      <td>0.195800</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>21.544347</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.867567</td>\n",
       "      <td>0.857512</td>\n",
       "      <td>0.852054</td>\n",
       "      <td>0.846883</td>\n",
       "      <td>0.873600</td>\n",
       "      <td>0.859523</td>\n",
       "      <td>0.009823</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>12.827075</td>\n",
       "      <td>0.430348</td>\n",
       "      <td>2.242276</td>\n",
       "      <td>0.205860</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>21.544347</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.863832</td>\n",
       "      <td>0.851767</td>\n",
       "      <td>0.846309</td>\n",
       "      <td>0.844872</td>\n",
       "      <td>0.865843</td>\n",
       "      <td>0.854525</td>\n",
       "      <td>0.008752</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>11.156855</td>\n",
       "      <td>0.467028</td>\n",
       "      <td>2.069154</td>\n",
       "      <td>0.159691</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>300</td>\n",
       "      <td>45</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.852054</td>\n",
       "      <td>0.844585</td>\n",
       "      <td>0.839989</td>\n",
       "      <td>0.833668</td>\n",
       "      <td>0.859523</td>\n",
       "      <td>0.845964</td>\n",
       "      <td>0.009052</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>12.062435</td>\n",
       "      <td>0.280006</td>\n",
       "      <td>2.068953</td>\n",
       "      <td>0.173405</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.864694</td>\n",
       "      <td>0.854352</td>\n",
       "      <td>0.850618</td>\n",
       "      <td>0.846596</td>\n",
       "      <td>0.867280</td>\n",
       "      <td>0.856708</td>\n",
       "      <td>0.008005</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "9       64.123026      3.714061         6.076129        1.974486   \n",
       "14      58.169902      0.920133         4.012292        0.195800   \n",
       "18      12.827075      0.430348         2.242276        0.205860   \n",
       "35      11.156855      0.467028         2.069154        0.159691   \n",
       "38      12.062435      0.280006         2.068953        0.173405   \n",
       "\n",
       "   param_vect__ngram_range param_vect__max_features param_svd__n_components  \\\n",
       "9                   (1, 2)                      400                      30   \n",
       "14                  (1, 2)                      400                      60   \n",
       "18                  (1, 1)                      400                      60   \n",
       "35                  (1, 1)                      300                      45   \n",
       "38                  (1, 1)                      400                      60   \n",
       "\n",
       "   param_log_clf__C                                             params  \\\n",
       "9           10000.0  {'vect__ngram_range': (1, 2), 'vect__max_featu...   \n",
       "14        21.544347  {'vect__ngram_range': (1, 2), 'vect__max_featu...   \n",
       "18        21.544347  {'vect__ngram_range': (1, 1), 'vect__max_featu...   \n",
       "35          10000.0  {'vect__ngram_range': (1, 1), 'vect__max_featu...   \n",
       "38          10000.0  {'vect__ngram_range': (1, 1), 'vect__max_featu...   \n",
       "\n",
       "    split0_test_score  split1_test_score  split2_test_score  \\\n",
       "9            0.848319           0.839127           0.827636   \n",
       "14           0.867567           0.857512           0.852054   \n",
       "18           0.863832           0.851767           0.846309   \n",
       "35           0.852054           0.844585           0.839989   \n",
       "38           0.864694           0.854352           0.850618   \n",
       "\n",
       "    split3_test_score  split4_test_score  mean_test_score  std_test_score  \\\n",
       "9            0.837690           0.850043         0.840563        0.008093   \n",
       "14           0.846883           0.873600         0.859523        0.009823   \n",
       "18           0.844872           0.865843         0.854525        0.008752   \n",
       "35           0.833668           0.859523         0.845964        0.009052   \n",
       "38           0.846596           0.867280         0.856708        0.008005   \n",
       "\n",
       "    rank_test_score  \n",
       "9                 5  \n",
       "14                1  \n",
       "18                3  \n",
       "35                4  \n",
       "38                2  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg_cv_results = bi.blosc_read(\"./data/log_grid_search_result.dat\")\n",
    "log_reg_top5 = log_reg_cv_results.query(\"rank_test_score <= 5\")\n",
    "log_reg_top5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9488a52a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.55      0.37      0.44      2097\n",
      "  right-wing       0.55      0.72      0.63      2255\n",
      "\n",
      "    accuracy                           0.55      4352\n",
      "   macro avg       0.55      0.55      0.53      4352\n",
      "weighted avg       0.55      0.55      0.54      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 780, 1317],\n",
       "       [ 631, 1624]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "# Loading Up the Same Pipeline as Random Search\n",
    "# Updated Values to Optimized\n",
    "log_clf = Pipeline([\n",
    "            ('vect', CountVectorizer()),\n",
    "            ('tfidf', TfidfTransformer()),\n",
    "            (\"svd\", TruncatedSVD()),\n",
    "            ('log_clf', LogisticRegression())\n",
    "        ])\n",
    "\n",
    "# Fitting the Optimized Log Reg Model\n",
    "log_clf.fit(x_train, y_train)\n",
    "\n",
    "# Getting Predictions on the testset\n",
    "predicted = log_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "# Printing the results\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "035255ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.87      0.84      0.85      2097\n",
      "  right-wing       0.85      0.88      0.87      2255\n",
      "\n",
      "    accuracy                           0.86      4352\n",
      "   macro avg       0.86      0.86      0.86      4352\n",
      "weighted avg       0.86      0.86      0.86      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1758,  339],\n",
       "       [ 272, 1983]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loading Up the Same Pipeline as Random Search\n",
    "# Updated Values to Optimized\n",
    "log_opt_clf = Pipeline([\n",
    "            ('vect', CountVectorizer(max_features=400, ngram_range=(1,2))),\n",
    "            ('tfidf', TfidfTransformer()),\n",
    "            (\"svd\", TruncatedSVD(n_components=60)),\n",
    "            ('log_clf', LogisticRegression(penalty=\"l1\", solver = 'saga', C=21.544347, max_iter=1000)), # this needs to be a different solver for LASSO\n",
    "        ])\n",
    "\n",
    "# Fitting the Optimized Log Reg Model\n",
    "log_opt_clf.fit(x_train, y_train)\n",
    "\n",
    "# Getting Predictions on the testset\n",
    "predicted = log_opt_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "# Printing the results\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b008ddf0",
   "metadata": {},
   "source": [
    "https://scikit-learn.org/stable/auto_examples/compose/plot_digits_pipe.html#sphx-glr-auto-examples-compose-plot-digits-pipe-py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "64bc099a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'past': 261,\n",
       " 'two': 371,\n",
       " 'years': 397,\n",
       " 'whether': 386,\n",
       " 'united': 373,\n",
       " 'states': 345,\n",
       " 'housing': 158,\n",
       " 'would': 395,\n",
       " 'financial': 116,\n",
       " 'economy': 98,\n",
       " 'prices': 281,\n",
       " 'first': 118,\n",
       " 'however': 159,\n",
       " 'may': 217,\n",
       " 'number': 244,\n",
       " 'process': 285,\n",
       " 'home': 156,\n",
       " 'income': 165,\n",
       " 'families': 112,\n",
       " 'five': 119,\n",
       " 'several': 332,\n",
       " 'high': 153,\n",
       " 'new': 238,\n",
       " 'york': 399,\n",
       " 'city': 54,\n",
       " 'washington': 382,\n",
       " '000': 0,\n",
       " 'according': 4,\n",
       " 'opportunity': 251,\n",
       " 'national': 231,\n",
       " 'less': 200,\n",
       " 'percent': 265,\n",
       " 'access': 3,\n",
       " 'house': 157,\n",
       " 'areas': 24,\n",
       " 'much': 228,\n",
       " 'media': 219,\n",
       " 'nation': 230,\n",
       " 'real': 295,\n",
       " 'data': 81,\n",
       " 'country': 73,\n",
       " 'still': 346,\n",
       " 'american': 17,\n",
       " 'second': 322,\n",
       " 'use': 376,\n",
       " 'development': 94,\n",
       " 'impact': 161,\n",
       " 'growth': 147,\n",
       " 'people': 263,\n",
       " 'market': 216,\n",
       " 'report': 304,\n",
       " 'problem': 283,\n",
       " 'could': 71,\n",
       " 'public': 289,\n",
       " 'policy': 272,\n",
       " 'future': 134,\n",
       " 'create': 78,\n",
       " 'low': 208,\n",
       " 'back': 28,\n",
       " 'although': 14,\n",
       " 'cost': 69,\n",
       " 'local': 206,\n",
       " 'increased': 167,\n",
       " 'government': 142,\n",
       " 'federal': 115,\n",
       " 'state': 344,\n",
       " 'programs': 287,\n",
       " 'means': 218,\n",
       " 'result': 308,\n",
       " 'across': 5,\n",
       " 'costs': 70,\n",
       " 'policies': 271,\n",
       " 'resources': 306,\n",
       " 'labor': 187,\n",
       " 'workers': 392,\n",
       " 'higher': 154,\n",
       " 'also': 13,\n",
       " 'global': 138,\n",
       " 'significant': 333,\n",
       " 'population': 276,\n",
       " 'major': 211,\n",
       " 'often': 248,\n",
       " 'continue': 67,\n",
       " 'political': 273,\n",
       " 'economic': 97,\n",
       " 'america': 16,\n",
       " 'rates': 293,\n",
       " 'changes': 49,\n",
       " 'four': 128,\n",
       " 'history': 155,\n",
       " 'early': 96,\n",
       " 'european': 107,\n",
       " 'europe': 106,\n",
       " 'center': 47,\n",
       " 'civil': 55,\n",
       " 'war': 381,\n",
       " 'south': 341,\n",
       " 'see': 325,\n",
       " 'movement': 227,\n",
       " 'many': 215,\n",
       " 'north': 242,\n",
       " 'better': 33,\n",
       " 'jobs': 183,\n",
       " 'within': 388,\n",
       " 'americans': 18,\n",
       " 'decades': 86,\n",
       " 'world': 394,\n",
       " 'another': 21,\n",
       " 'take': 353,\n",
       " 'lower': 209,\n",
       " 'change': 48,\n",
       " 'congress': 65,\n",
       " 'services': 330,\n",
       " 'likely': 204,\n",
       " 'response': 307,\n",
       " 'among': 19,\n",
       " 'majority': 212,\n",
       " 'free': 129,\n",
       " 'oil': 249,\n",
       " 'later': 190,\n",
       " 'led': 196,\n",
       " 'power': 279,\n",
       " 'supreme': 350,\n",
       " 'court': 75,\n",
       " 'common': 61,\n",
       " 'rights': 310,\n",
       " 'health': 148,\n",
       " 'issues': 181,\n",
       " 'example': 110,\n",
       " 'and': 20,\n",
       " 'middle': 222,\n",
       " 'anti': 22,\n",
       " 'using': 378,\n",
       " 'education': 99,\n",
       " 'tax': 354,\n",
       " 'social': 338,\n",
       " 'called': 41,\n",
       " 'one': 250,\n",
       " 'per': 264,\n",
       " 'potential': 278,\n",
       " 'set': 331,\n",
       " 'making': 214,\n",
       " 'community': 62,\n",
       " 'society': 339,\n",
       " 'course': 74,\n",
       " 'increase': 166,\n",
       " 'include': 163,\n",
       " 'form': 124,\n",
       " 'countries': 72,\n",
       " 'recent': 298,\n",
       " 'force': 121,\n",
       " 'place': 266,\n",
       " 'must': 229,\n",
       " 'order': 252,\n",
       " 'provide': 288,\n",
       " 'instead': 172,\n",
       " 'private': 282,\n",
       " 'authority': 26,\n",
       " 'way': 383,\n",
       " 'least': 195,\n",
       " 'even': 108,\n",
       " 'though': 360,\n",
       " 'average': 27,\n",
       " 'nearly': 233,\n",
       " 'small': 337,\n",
       " 'best': 32,\n",
       " 'large': 188,\n",
       " 'times': 365,\n",
       " 'next': 240,\n",
       " 'laws': 192,\n",
       " 'able': 2,\n",
       " 'full': 131,\n",
       " 'similar': 334,\n",
       " 'used': 377,\n",
       " 'terms': 357,\n",
       " 'region': 302,\n",
       " 'poor': 275,\n",
       " 'year': 396,\n",
       " 'total': 367,\n",
       " 'plan': 267,\n",
       " 'since': 336,\n",
       " 'billion': 37,\n",
       " 'found': 126,\n",
       " 'others': 255,\n",
       " 'cases': 46,\n",
       " '19': 1,\n",
       " 'family': 113,\n",
       " 'end': 102,\n",
       " 'spending': 343,\n",
       " 'benefits': 31,\n",
       " 'three': 362,\n",
       " 'rate': 292,\n",
       " 'school': 320,\n",
       " 'great': 144,\n",
       " 'find': 117,\n",
       " 'work': 391,\n",
       " 'life': 202,\n",
       " 'individuals': 169,\n",
       " 'part': 257,\n",
       " 'general': 135,\n",
       " 'black': 38,\n",
       " 'white': 387,\n",
       " 'yet': 398,\n",
       " 'every': 109,\n",
       " 'business': 40,\n",
       " 'today': 366,\n",
       " 'given': 137,\n",
       " 'current': 80,\n",
       " 'level': 201,\n",
       " 'program': 286,\n",
       " 'pay': 262,\n",
       " 'time': 364,\n",
       " 'problems': 284,\n",
       " 'become': 30,\n",
       " 'including': 164,\n",
       " 'important': 162,\n",
       " 'foreign': 123,\n",
       " 'million': 225,\n",
       " 'especially': 105,\n",
       " 'long': 207,\n",
       " 'made': 210,\n",
       " 'possible': 277,\n",
       " 'might': 223,\n",
       " 'little': 205,\n",
       " 'group': 145,\n",
       " 'thus': 363,\n",
       " 'groups': 146,\n",
       " 'working': 393,\n",
       " 'decision': 87,\n",
       " 'budget': 39,\n",
       " 'based': 29,\n",
       " 'almost': 11,\n",
       " 'de': 83,\n",
       " 'without': 389,\n",
       " 'put': 290,\n",
       " 'companies': 63,\n",
       " 'money': 226,\n",
       " 'despite': 93,\n",
       " 'things': 358,\n",
       " 'care': 44,\n",
       " 'individual': 168,\n",
       " 'trade': 369,\n",
       " 'job': 182,\n",
       " 'research': 305,\n",
       " 'going': 140,\n",
       " 'case': 45,\n",
       " 'union': 372,\n",
       " 'rules': 314,\n",
       " 'day': 82,\n",
       " 'like': 203,\n",
       " 'interest': 175,\n",
       " 'make': 213,\n",
       " 'efforts': 100,\n",
       " 'need': 235,\n",
       " 'heritage': 151,\n",
       " 'foundation': 127,\n",
       " 'united states': 374,\n",
       " 'new york': 239,\n",
       " 'supreme court': 351,\n",
       " 'heritage foundation': 152,\n",
       " 'key': 185,\n",
       " 'politics': 274,\n",
       " 'organization': 253,\n",
       " 'risk': 311,\n",
       " 'members': 221,\n",
       " 'party': 260,\n",
       " 'children': 50,\n",
       " 'democratic': 90,\n",
       " 'always': 15,\n",
       " 'class': 56,\n",
       " 'self': 326,\n",
       " 'legal': 198,\n",
       " 'left': 197,\n",
       " 'never': 237,\n",
       " 'sense': 328,\n",
       " 'welfare': 384,\n",
       " 'russian': 316,\n",
       " 'russia': 315,\n",
       " 'interests': 176,\n",
       " 'strategy': 347,\n",
       " 'rather': 294,\n",
       " 'well': 385,\n",
       " 'right': 309,\n",
       " 'non': 241,\n",
       " 'control': 68,\n",
       " 'simply': 335,\n",
       " 'point': 269,\n",
       " 'get': 136,\n",
       " 'human': 160,\n",
       " 'issue': 180,\n",
       " 'fact': 111,\n",
       " 'role': 312,\n",
       " 'leaders': 193,\n",
       " 'rule': 313,\n",
       " 'religious': 303,\n",
       " 'far': 114,\n",
       " 'department': 92,\n",
       " 'governments': 143,\n",
       " 'crisis': 79,\n",
       " 'plans': 268,\n",
       " 'particularly': 258,\n",
       " 'already': 12,\n",
       " 'forces': 122,\n",
       " 'come': 59,\n",
       " 'support': 349,\n",
       " 'want': 380,\n",
       " 'system': 352,\n",
       " 'us': 375,\n",
       " 'nations': 232,\n",
       " 'agreement': 10,\n",
       " 'industry': 170,\n",
       " 'defense': 88,\n",
       " 'think': 359,\n",
       " 'international': 177,\n",
       " 'vote': 379,\n",
       " 'committee': 60,\n",
       " 'something': 340,\n",
       " 'administration': 9,\n",
       " 'military': 224,\n",
       " 'justice': 184,\n",
       " 'different': 95,\n",
       " 'approach': 23,\n",
       " 'clear': 57,\n",
       " 'parties': 259,\n",
       " 'insurance': 174,\n",
       " 'good': 141,\n",
       " 'go': 139,\n",
       " 'women': 390,\n",
       " 'leadership': 194,\n",
       " 'soviet': 342,\n",
       " 'deal': 84,\n",
       " 'around': 25,\n",
       " 'funds': 133,\n",
       " 'institutions': 173,\n",
       " 'legislation': 199,\n",
       " 'organizations': 254,\n",
       " 'act': 6,\n",
       " 'actually': 8,\n",
       " 'last': 189,\n",
       " 'democracy': 89,\n",
       " 'term': 356,\n",
       " 'question': 291,\n",
       " 'students': 348,\n",
       " 'former': 125,\n",
       " 'trump': 370,\n",
       " 'said': 317,\n",
       " 'schools': 321,\n",
       " 'enough': 104,\n",
       " 'law': 191,\n",
       " 'citizens': 53,\n",
       " 'covid': 76,\n",
       " 'pandemic': 256,\n",
       " 'help': 150,\n",
       " 'big': 35,\n",
       " 'office': 246,\n",
       " 'covid 19': 77,\n",
       " 'say': 319,\n",
       " 'regime': 301,\n",
       " 'capital': 43,\n",
       " 'freedom': 130,\n",
       " 'democrats': 91,\n",
       " 'service': 329,\n",
       " 'campaign': 42,\n",
       " 'toward': 368,\n",
       " 'election': 101,\n",
       " 'constitution': 66,\n",
       " 'action': 7,\n",
       " 'reform': 300,\n",
       " 'health care': 149,\n",
       " 'china': 51,\n",
       " 'nuclear': 243,\n",
       " 'information': 171,\n",
       " 'chinese': 52,\n",
       " 'threat': 361,\n",
       " 'needs': 236,\n",
       " 'biden': 34,\n",
       " 'investment': 178,\n",
       " 'company': 64,\n",
       " 'security': 324,\n",
       " 'police': 270,\n",
       " 'sanders': 318,\n",
       " 'know': 186,\n",
       " 'president': 280,\n",
       " 'bill': 36,\n",
       " 'obama': 245,\n",
       " 'sector': 323,\n",
       " 'food': 120,\n",
       " 'necessary': 234,\n",
       " 'reason': 297,\n",
       " 'really': 296,\n",
       " 'debt': 85,\n",
       " 'senate': 327,\n",
       " 'funding': 132,\n",
       " 'officials': 247,\n",
       " 'iran': 179,\n",
       " 'taxes': 355,\n",
       " 'energy': 103,\n",
       " 'climate': 58,\n",
       " 'medicare': 220,\n",
       " 'ref': 299}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_opt_clf[0].vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c5098467",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00999342, 0.02655163, 0.02587907, 0.02104172, 0.01868057,\n",
       "       0.01556065, 0.01453942, 0.0141053 , 0.01339169, 0.01308897,\n",
       "       0.01171164, 0.01129167, 0.01045946, 0.01028872, 0.00970334,\n",
       "       0.00897715, 0.00843319, 0.00827448, 0.00811224, 0.00785614,\n",
       "       0.0074517 , 0.00715055, 0.00712232, 0.00679074, 0.00656477,\n",
       "       0.00647838, 0.00628882, 0.00617062, 0.00594708, 0.00585181,\n",
       "       0.0055752 , 0.00553885, 0.00534434, 0.00523186, 0.00507057,\n",
       "       0.00498511, 0.00495369, 0.00473667, 0.00470005, 0.00458031,\n",
       "       0.0045251 , 0.00449452, 0.00426861, 0.00418954, 0.00411882,\n",
       "       0.00404057, 0.00403422, 0.00399984, 0.00389708, 0.00385365,\n",
       "       0.0037711 , 0.00372043, 0.0036199 , 0.00357318, 0.00356906,\n",
       "       0.00354796, 0.00339719, 0.003383  , 0.00334645, 0.00325617])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_opt_clf[2].explained_variance_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5896b741",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_vect__ngram_range</th>\n",
       "      <th>param_vect__max_features</th>\n",
       "      <th>param_svd__n_components</th>\n",
       "      <th>param_svc_clf__penalty</th>\n",
       "      <th>param_svc_clf__C</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9.538747</td>\n",
       "      <td>0.252239</td>\n",
       "      <td>1.554701</td>\n",
       "      <td>0.083288</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>400</td>\n",
       "      <td>45</td>\n",
       "      <td>l2</td>\n",
       "      <td>21.544347</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.857799</td>\n",
       "      <td>0.847745</td>\n",
       "      <td>0.841712</td>\n",
       "      <td>0.840563</td>\n",
       "      <td>0.862396</td>\n",
       "      <td>0.850043</td>\n",
       "      <td>0.008687</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>8.757007</td>\n",
       "      <td>0.317715</td>\n",
       "      <td>1.550521</td>\n",
       "      <td>0.073838</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>200</td>\n",
       "      <td>60</td>\n",
       "      <td>l2</td>\n",
       "      <td>0.046416</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.846021</td>\n",
       "      <td>0.832232</td>\n",
       "      <td>0.829934</td>\n",
       "      <td>0.826774</td>\n",
       "      <td>0.843149</td>\n",
       "      <td>0.835622</td>\n",
       "      <td>0.007575</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>40.221779</td>\n",
       "      <td>1.873627</td>\n",
       "      <td>3.265838</td>\n",
       "      <td>0.243435</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>45</td>\n",
       "      <td>l2</td>\n",
       "      <td>0.046416</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.860672</td>\n",
       "      <td>0.847745</td>\n",
       "      <td>0.841999</td>\n",
       "      <td>0.840850</td>\n",
       "      <td>0.860385</td>\n",
       "      <td>0.850330</td>\n",
       "      <td>0.008649</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>10.407523</td>\n",
       "      <td>0.389623</td>\n",
       "      <td>1.543918</td>\n",
       "      <td>0.103774</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>300</td>\n",
       "      <td>60</td>\n",
       "      <td>l2</td>\n",
       "      <td>21.544347</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.853778</td>\n",
       "      <td>0.847170</td>\n",
       "      <td>0.849756</td>\n",
       "      <td>0.838839</td>\n",
       "      <td>0.860672</td>\n",
       "      <td>0.850043</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>42.016662</td>\n",
       "      <td>1.504122</td>\n",
       "      <td>2.994632</td>\n",
       "      <td>0.116347</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>200</td>\n",
       "      <td>60</td>\n",
       "      <td>l2</td>\n",
       "      <td>21.544347</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.842861</td>\n",
       "      <td>0.833668</td>\n",
       "      <td>0.832519</td>\n",
       "      <td>0.822465</td>\n",
       "      <td>0.848607</td>\n",
       "      <td>0.836024</td>\n",
       "      <td>0.009019</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "5        9.538747      0.252239         1.554701        0.083288   \n",
       "12       8.757007      0.317715         1.550521        0.073838   \n",
       "14      40.221779      1.873627         3.265838        0.243435   \n",
       "24      10.407523      0.389623         1.543918        0.103774   \n",
       "39      42.016662      1.504122         2.994632        0.116347   \n",
       "\n",
       "   param_vect__ngram_range param_vect__max_features param_svd__n_components  \\\n",
       "5                   (1, 1)                      400                      45   \n",
       "12                  (1, 1)                      200                      60   \n",
       "14                  (1, 2)                      400                      45   \n",
       "24                  (1, 1)                      300                      60   \n",
       "39                  (1, 2)                      200                      60   \n",
       "\n",
       "   param_svc_clf__penalty param_svc_clf__C  \\\n",
       "5                      l2        21.544347   \n",
       "12                     l2         0.046416   \n",
       "14                     l2         0.046416   \n",
       "24                     l2        21.544347   \n",
       "39                     l2        21.544347   \n",
       "\n",
       "                                               params  split0_test_score  \\\n",
       "5   {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.857799   \n",
       "12  {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.846021   \n",
       "14  {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.860672   \n",
       "24  {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.853778   \n",
       "39  {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.842861   \n",
       "\n",
       "    split1_test_score  split2_test_score  split3_test_score  \\\n",
       "5            0.847745           0.841712           0.840563   \n",
       "12           0.832232           0.829934           0.826774   \n",
       "14           0.847745           0.841999           0.840850   \n",
       "24           0.847170           0.849756           0.838839   \n",
       "39           0.833668           0.832519           0.822465   \n",
       "\n",
       "    split4_test_score  mean_test_score  std_test_score  rank_test_score  \n",
       "5            0.862396         0.850043        0.008687                2  \n",
       "12           0.843149         0.835622        0.007575                5  \n",
       "14           0.860385         0.850330        0.008649                1  \n",
       "24           0.860672         0.850043        0.007222                2  \n",
       "39           0.848607         0.836024        0.009019                4  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc_reg_cv_results = bi.blosc_read(\"./data/svc_grid_search_result.dat\")\n",
    "svc_top5 = svc_reg_cv_results.query(\"rank_test_score <= 5\")\n",
    "svc_top5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cbc1cf6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.55      0.37      0.45      2097\n",
      "  right-wing       0.55      0.72      0.63      2255\n",
      "\n",
      "    accuracy                           0.55      4352\n",
      "   macro avg       0.55      0.55      0.54      4352\n",
      "weighted avg       0.55      0.55      0.54      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 783, 1314],\n",
       "       [ 632, 1623]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "svc_clf = Pipeline([\n",
    "            ('vect', CountVectorizer()),\n",
    "            ('tfidf', TfidfTransformer()),\n",
    "            (\"svd\", TruncatedSVD()),\n",
    "            ('svc_clf', LinearSVC()),\n",
    "        ])\n",
    "\n",
    "svc_clf.fit(x_train, y_train)\n",
    "\n",
    "predicted = svc_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f3532ecb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.86      0.83      0.84      2097\n",
      "  right-wing       0.85      0.88      0.86      2255\n",
      "\n",
      "    accuracy                           0.85      4352\n",
      "   macro avg       0.85      0.85      0.85      4352\n",
      "weighted avg       0.85      0.85      0.85      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1735,  362],\n",
       "       [ 276, 1979]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc_opt_clf = Pipeline([\n",
    "            ('vect', CountVectorizer(max_features=400, ngram_range=(1,2))),\n",
    "            ('tfidf', TfidfTransformer()),\n",
    "            (\"svd\", TruncatedSVD(n_components=45)),\n",
    "            ('svc_clf', LinearSVC(penalty = \"l2\", C = 0.046415)),\n",
    "        ])\n",
    "\n",
    "svc_opt_clf.fit(x_train, y_train)\n",
    "\n",
    "predicted = svc_opt_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "18ae098f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'past': 261,\n",
       " 'two': 371,\n",
       " 'years': 397,\n",
       " 'whether': 386,\n",
       " 'united': 373,\n",
       " 'states': 345,\n",
       " 'housing': 158,\n",
       " 'would': 395,\n",
       " 'financial': 116,\n",
       " 'economy': 98,\n",
       " 'prices': 281,\n",
       " 'first': 118,\n",
       " 'however': 159,\n",
       " 'may': 217,\n",
       " 'number': 244,\n",
       " 'process': 285,\n",
       " 'home': 156,\n",
       " 'income': 165,\n",
       " 'families': 112,\n",
       " 'five': 119,\n",
       " 'several': 332,\n",
       " 'high': 153,\n",
       " 'new': 238,\n",
       " 'york': 399,\n",
       " 'city': 54,\n",
       " 'washington': 382,\n",
       " '000': 0,\n",
       " 'according': 4,\n",
       " 'opportunity': 251,\n",
       " 'national': 231,\n",
       " 'less': 200,\n",
       " 'percent': 265,\n",
       " 'access': 3,\n",
       " 'house': 157,\n",
       " 'areas': 24,\n",
       " 'much': 228,\n",
       " 'media': 219,\n",
       " 'nation': 230,\n",
       " 'real': 295,\n",
       " 'data': 81,\n",
       " 'country': 73,\n",
       " 'still': 346,\n",
       " 'american': 17,\n",
       " 'second': 322,\n",
       " 'use': 376,\n",
       " 'development': 94,\n",
       " 'impact': 161,\n",
       " 'growth': 147,\n",
       " 'people': 263,\n",
       " 'market': 216,\n",
       " 'report': 304,\n",
       " 'problem': 283,\n",
       " 'could': 71,\n",
       " 'public': 289,\n",
       " 'policy': 272,\n",
       " 'future': 134,\n",
       " 'create': 78,\n",
       " 'low': 208,\n",
       " 'back': 28,\n",
       " 'although': 14,\n",
       " 'cost': 69,\n",
       " 'local': 206,\n",
       " 'increased': 167,\n",
       " 'government': 142,\n",
       " 'federal': 115,\n",
       " 'state': 344,\n",
       " 'programs': 287,\n",
       " 'means': 218,\n",
       " 'result': 308,\n",
       " 'across': 5,\n",
       " 'costs': 70,\n",
       " 'policies': 271,\n",
       " 'resources': 306,\n",
       " 'labor': 187,\n",
       " 'workers': 392,\n",
       " 'higher': 154,\n",
       " 'also': 13,\n",
       " 'global': 138,\n",
       " 'significant': 333,\n",
       " 'population': 276,\n",
       " 'major': 211,\n",
       " 'often': 248,\n",
       " 'continue': 67,\n",
       " 'political': 273,\n",
       " 'economic': 97,\n",
       " 'america': 16,\n",
       " 'rates': 293,\n",
       " 'changes': 49,\n",
       " 'four': 128,\n",
       " 'history': 155,\n",
       " 'early': 96,\n",
       " 'european': 107,\n",
       " 'europe': 106,\n",
       " 'center': 47,\n",
       " 'civil': 55,\n",
       " 'war': 381,\n",
       " 'south': 341,\n",
       " 'see': 325,\n",
       " 'movement': 227,\n",
       " 'many': 215,\n",
       " 'north': 242,\n",
       " 'better': 33,\n",
       " 'jobs': 183,\n",
       " 'within': 388,\n",
       " 'americans': 18,\n",
       " 'decades': 86,\n",
       " 'world': 394,\n",
       " 'another': 21,\n",
       " 'take': 353,\n",
       " 'lower': 209,\n",
       " 'change': 48,\n",
       " 'congress': 65,\n",
       " 'services': 330,\n",
       " 'likely': 204,\n",
       " 'response': 307,\n",
       " 'among': 19,\n",
       " 'majority': 212,\n",
       " 'free': 129,\n",
       " 'oil': 249,\n",
       " 'later': 190,\n",
       " 'led': 196,\n",
       " 'power': 279,\n",
       " 'supreme': 350,\n",
       " 'court': 75,\n",
       " 'common': 61,\n",
       " 'rights': 310,\n",
       " 'health': 148,\n",
       " 'issues': 181,\n",
       " 'example': 110,\n",
       " 'and': 20,\n",
       " 'middle': 222,\n",
       " 'anti': 22,\n",
       " 'using': 378,\n",
       " 'education': 99,\n",
       " 'tax': 354,\n",
       " 'social': 338,\n",
       " 'called': 41,\n",
       " 'one': 250,\n",
       " 'per': 264,\n",
       " 'potential': 278,\n",
       " 'set': 331,\n",
       " 'making': 214,\n",
       " 'community': 62,\n",
       " 'society': 339,\n",
       " 'course': 74,\n",
       " 'increase': 166,\n",
       " 'include': 163,\n",
       " 'form': 124,\n",
       " 'countries': 72,\n",
       " 'recent': 298,\n",
       " 'force': 121,\n",
       " 'place': 266,\n",
       " 'must': 229,\n",
       " 'order': 252,\n",
       " 'provide': 288,\n",
       " 'instead': 172,\n",
       " 'private': 282,\n",
       " 'authority': 26,\n",
       " 'way': 383,\n",
       " 'least': 195,\n",
       " 'even': 108,\n",
       " 'though': 360,\n",
       " 'average': 27,\n",
       " 'nearly': 233,\n",
       " 'small': 337,\n",
       " 'best': 32,\n",
       " 'large': 188,\n",
       " 'times': 365,\n",
       " 'next': 240,\n",
       " 'laws': 192,\n",
       " 'able': 2,\n",
       " 'full': 131,\n",
       " 'similar': 334,\n",
       " 'used': 377,\n",
       " 'terms': 357,\n",
       " 'region': 302,\n",
       " 'poor': 275,\n",
       " 'year': 396,\n",
       " 'total': 367,\n",
       " 'plan': 267,\n",
       " 'since': 336,\n",
       " 'billion': 37,\n",
       " 'found': 126,\n",
       " 'others': 255,\n",
       " 'cases': 46,\n",
       " '19': 1,\n",
       " 'family': 113,\n",
       " 'end': 102,\n",
       " 'spending': 343,\n",
       " 'benefits': 31,\n",
       " 'three': 362,\n",
       " 'rate': 292,\n",
       " 'school': 320,\n",
       " 'great': 144,\n",
       " 'find': 117,\n",
       " 'work': 391,\n",
       " 'life': 202,\n",
       " 'individuals': 169,\n",
       " 'part': 257,\n",
       " 'general': 135,\n",
       " 'black': 38,\n",
       " 'white': 387,\n",
       " 'yet': 398,\n",
       " 'every': 109,\n",
       " 'business': 40,\n",
       " 'today': 366,\n",
       " 'given': 137,\n",
       " 'current': 80,\n",
       " 'level': 201,\n",
       " 'program': 286,\n",
       " 'pay': 262,\n",
       " 'time': 364,\n",
       " 'problems': 284,\n",
       " 'become': 30,\n",
       " 'including': 164,\n",
       " 'important': 162,\n",
       " 'foreign': 123,\n",
       " 'million': 225,\n",
       " 'especially': 105,\n",
       " 'long': 207,\n",
       " 'made': 210,\n",
       " 'possible': 277,\n",
       " 'might': 223,\n",
       " 'little': 205,\n",
       " 'group': 145,\n",
       " 'thus': 363,\n",
       " 'groups': 146,\n",
       " 'working': 393,\n",
       " 'decision': 87,\n",
       " 'budget': 39,\n",
       " 'based': 29,\n",
       " 'almost': 11,\n",
       " 'de': 83,\n",
       " 'without': 389,\n",
       " 'put': 290,\n",
       " 'companies': 63,\n",
       " 'money': 226,\n",
       " 'despite': 93,\n",
       " 'things': 358,\n",
       " 'care': 44,\n",
       " 'individual': 168,\n",
       " 'trade': 369,\n",
       " 'job': 182,\n",
       " 'research': 305,\n",
       " 'going': 140,\n",
       " 'case': 45,\n",
       " 'union': 372,\n",
       " 'rules': 314,\n",
       " 'day': 82,\n",
       " 'like': 203,\n",
       " 'interest': 175,\n",
       " 'make': 213,\n",
       " 'efforts': 100,\n",
       " 'need': 235,\n",
       " 'heritage': 151,\n",
       " 'foundation': 127,\n",
       " 'united states': 374,\n",
       " 'new york': 239,\n",
       " 'supreme court': 351,\n",
       " 'heritage foundation': 152,\n",
       " 'key': 185,\n",
       " 'politics': 274,\n",
       " 'organization': 253,\n",
       " 'risk': 311,\n",
       " 'members': 221,\n",
       " 'party': 260,\n",
       " 'children': 50,\n",
       " 'democratic': 90,\n",
       " 'always': 15,\n",
       " 'class': 56,\n",
       " 'self': 326,\n",
       " 'legal': 198,\n",
       " 'left': 197,\n",
       " 'never': 237,\n",
       " 'sense': 328,\n",
       " 'welfare': 384,\n",
       " 'russian': 316,\n",
       " 'russia': 315,\n",
       " 'interests': 176,\n",
       " 'strategy': 347,\n",
       " 'rather': 294,\n",
       " 'well': 385,\n",
       " 'right': 309,\n",
       " 'non': 241,\n",
       " 'control': 68,\n",
       " 'simply': 335,\n",
       " 'point': 269,\n",
       " 'get': 136,\n",
       " 'human': 160,\n",
       " 'issue': 180,\n",
       " 'fact': 111,\n",
       " 'role': 312,\n",
       " 'leaders': 193,\n",
       " 'rule': 313,\n",
       " 'religious': 303,\n",
       " 'far': 114,\n",
       " 'department': 92,\n",
       " 'governments': 143,\n",
       " 'crisis': 79,\n",
       " 'plans': 268,\n",
       " 'particularly': 258,\n",
       " 'already': 12,\n",
       " 'forces': 122,\n",
       " 'come': 59,\n",
       " 'support': 349,\n",
       " 'want': 380,\n",
       " 'system': 352,\n",
       " 'us': 375,\n",
       " 'nations': 232,\n",
       " 'agreement': 10,\n",
       " 'industry': 170,\n",
       " 'defense': 88,\n",
       " 'think': 359,\n",
       " 'international': 177,\n",
       " 'vote': 379,\n",
       " 'committee': 60,\n",
       " 'something': 340,\n",
       " 'administration': 9,\n",
       " 'military': 224,\n",
       " 'justice': 184,\n",
       " 'different': 95,\n",
       " 'approach': 23,\n",
       " 'clear': 57,\n",
       " 'parties': 259,\n",
       " 'insurance': 174,\n",
       " 'good': 141,\n",
       " 'go': 139,\n",
       " 'women': 390,\n",
       " 'leadership': 194,\n",
       " 'soviet': 342,\n",
       " 'deal': 84,\n",
       " 'around': 25,\n",
       " 'funds': 133,\n",
       " 'institutions': 173,\n",
       " 'legislation': 199,\n",
       " 'organizations': 254,\n",
       " 'act': 6,\n",
       " 'actually': 8,\n",
       " 'last': 189,\n",
       " 'democracy': 89,\n",
       " 'term': 356,\n",
       " 'question': 291,\n",
       " 'students': 348,\n",
       " 'former': 125,\n",
       " 'trump': 370,\n",
       " 'said': 317,\n",
       " 'schools': 321,\n",
       " 'enough': 104,\n",
       " 'law': 191,\n",
       " 'citizens': 53,\n",
       " 'covid': 76,\n",
       " 'pandemic': 256,\n",
       " 'help': 150,\n",
       " 'big': 35,\n",
       " 'office': 246,\n",
       " 'covid 19': 77,\n",
       " 'say': 319,\n",
       " 'regime': 301,\n",
       " 'capital': 43,\n",
       " 'freedom': 130,\n",
       " 'democrats': 91,\n",
       " 'service': 329,\n",
       " 'campaign': 42,\n",
       " 'toward': 368,\n",
       " 'election': 101,\n",
       " 'constitution': 66,\n",
       " 'action': 7,\n",
       " 'reform': 300,\n",
       " 'health care': 149,\n",
       " 'china': 51,\n",
       " 'nuclear': 243,\n",
       " 'information': 171,\n",
       " 'chinese': 52,\n",
       " 'threat': 361,\n",
       " 'needs': 236,\n",
       " 'biden': 34,\n",
       " 'investment': 178,\n",
       " 'company': 64,\n",
       " 'security': 324,\n",
       " 'police': 270,\n",
       " 'sanders': 318,\n",
       " 'know': 186,\n",
       " 'president': 280,\n",
       " 'bill': 36,\n",
       " 'obama': 245,\n",
       " 'sector': 323,\n",
       " 'food': 120,\n",
       " 'necessary': 234,\n",
       " 'reason': 297,\n",
       " 'really': 296,\n",
       " 'debt': 85,\n",
       " 'senate': 327,\n",
       " 'funding': 132,\n",
       " 'officials': 247,\n",
       " 'iran': 179,\n",
       " 'taxes': 355,\n",
       " 'energy': 103,\n",
       " 'climate': 58,\n",
       " 'medicare': 220,\n",
       " 'ref': 299}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc_opt_clf[0].vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "39a79ed0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00999342, 0.02655163, 0.02587907, 0.02104172, 0.01868057,\n",
       "       0.01556065, 0.01453942, 0.0141053 , 0.01339169, 0.01308897,\n",
       "       0.01171164, 0.01129166, 0.01045944, 0.0102887 , 0.00970333,\n",
       "       0.00897707, 0.00843297, 0.00827445, 0.00811203, 0.00785521,\n",
       "       0.00745135, 0.0071496 , 0.00712206, 0.00678999, 0.00656446,\n",
       "       0.00647703, 0.00628619, 0.0061665 , 0.00594519, 0.00584741,\n",
       "       0.0055662 , 0.00552201, 0.00533001, 0.00522641, 0.00503176,\n",
       "       0.00497706, 0.00491041, 0.00470204, 0.00464394, 0.00454361,\n",
       "       0.00448433, 0.00446361, 0.00420491, 0.00409596, 0.0040299 ])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc_opt_clf[2].explained_variance_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ad385df0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_vect__ngram_range</th>\n",
       "      <th>param_vect__max_features</th>\n",
       "      <th>param_svd__n_components</th>\n",
       "      <th>param_knn_clf__n_neighbors</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.708155</td>\n",
       "      <td>0.123026</td>\n",
       "      <td>2.061593</td>\n",
       "      <td>0.065220</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.858661</td>\n",
       "      <td>0.861247</td>\n",
       "      <td>0.856076</td>\n",
       "      <td>0.850618</td>\n",
       "      <td>0.863545</td>\n",
       "      <td>0.858029</td>\n",
       "      <td>0.004470</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>38.017391</td>\n",
       "      <td>0.497011</td>\n",
       "      <td>3.561517</td>\n",
       "      <td>0.061605</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>300</td>\n",
       "      <td>45</td>\n",
       "      <td>6</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.845734</td>\n",
       "      <td>0.855214</td>\n",
       "      <td>0.853203</td>\n",
       "      <td>0.838552</td>\n",
       "      <td>0.848319</td>\n",
       "      <td>0.848205</td>\n",
       "      <td>0.005889</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>8.183979</td>\n",
       "      <td>0.187956</td>\n",
       "      <td>2.308006</td>\n",
       "      <td>0.287278</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>300</td>\n",
       "      <td>45</td>\n",
       "      <td>5</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.849181</td>\n",
       "      <td>0.852054</td>\n",
       "      <td>0.853778</td>\n",
       "      <td>0.843149</td>\n",
       "      <td>0.845159</td>\n",
       "      <td>0.848664</td>\n",
       "      <td>0.004015</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>38.280288</td>\n",
       "      <td>0.235090</td>\n",
       "      <td>3.726631</td>\n",
       "      <td>0.112006</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>45</td>\n",
       "      <td>6</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.846021</td>\n",
       "      <td>0.851767</td>\n",
       "      <td>0.858949</td>\n",
       "      <td>0.851767</td>\n",
       "      <td>0.856938</td>\n",
       "      <td>0.853088</td>\n",
       "      <td>0.004530</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>8.004029</td>\n",
       "      <td>0.397554</td>\n",
       "      <td>2.160517</td>\n",
       "      <td>0.036752</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>300</td>\n",
       "      <td>60</td>\n",
       "      <td>6</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.852629</td>\n",
       "      <td>0.849469</td>\n",
       "      <td>0.852341</td>\n",
       "      <td>0.836541</td>\n",
       "      <td>0.842861</td>\n",
       "      <td>0.846768</td>\n",
       "      <td>0.006206</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "2        7.708155      0.123026         2.061593        0.065220   \n",
       "7       38.017391      0.497011         3.561517        0.061605   \n",
       "10       8.183979      0.187956         2.308006        0.287278   \n",
       "27      38.280288      0.235090         3.726631        0.112006   \n",
       "35       8.004029      0.397554         2.160517        0.036752   \n",
       "\n",
       "   param_vect__ngram_range param_vect__max_features param_svd__n_components  \\\n",
       "2                   (1, 1)                      400                      60   \n",
       "7                   (1, 2)                      300                      45   \n",
       "10                  (1, 1)                      300                      45   \n",
       "27                  (1, 2)                      400                      45   \n",
       "35                  (1, 1)                      300                      60   \n",
       "\n",
       "   param_knn_clf__n_neighbors  \\\n",
       "2                           5   \n",
       "7                           6   \n",
       "10                          5   \n",
       "27                          6   \n",
       "35                          6   \n",
       "\n",
       "                                               params  split0_test_score  \\\n",
       "2   {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.858661   \n",
       "7   {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.845734   \n",
       "10  {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.849181   \n",
       "27  {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.846021   \n",
       "35  {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.852629   \n",
       "\n",
       "    split1_test_score  split2_test_score  split3_test_score  \\\n",
       "2            0.861247           0.856076           0.850618   \n",
       "7            0.855214           0.853203           0.838552   \n",
       "10           0.852054           0.853778           0.843149   \n",
       "27           0.851767           0.858949           0.851767   \n",
       "35           0.849469           0.852341           0.836541   \n",
       "\n",
       "    split4_test_score  mean_test_score  std_test_score  rank_test_score  \n",
       "2            0.863545         0.858029        0.004470                1  \n",
       "7            0.848319         0.848205        0.005889                4  \n",
       "10           0.845159         0.848664        0.004015                3  \n",
       "27           0.856938         0.853088        0.004530                2  \n",
       "35           0.842861         0.846768        0.006206                5  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_reg_cv_results = bi.blosc_read(\"./data/knn_grid_search_result.dat\")\n",
    "knn_top5 = knn_reg_cv_results.query(\"rank_test_score <= 5\")\n",
    "knn_top5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fbd20b61",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.54      0.53      0.53      2097\n",
      "  right-wing       0.57      0.58      0.58      2255\n",
      "\n",
      "    accuracy                           0.56      4352\n",
      "   macro avg       0.55      0.55      0.55      4352\n",
      "weighted avg       0.55      0.56      0.55      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1103,  994],\n",
       "       [ 942, 1313]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn_clf = Pipeline([\n",
    "        ('vect', CountVectorizer()),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        (\"svd\", TruncatedSVD()),\n",
    "        ('knn_clf', KNeighborsClassifier()),\n",
    "    ])\n",
    "\n",
    "knn_clf.fit(x_train, y_train)\n",
    "\n",
    "predicted = knn_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2df58e81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.88      0.82      0.85      2097\n",
      "  right-wing       0.84      0.90      0.87      2255\n",
      "\n",
      "    accuracy                           0.86      4352\n",
      "   macro avg       0.86      0.86      0.86      4352\n",
      "weighted avg       0.86      0.86      0.86      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1725,  372],\n",
       "       [ 235, 2020]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_opt_clf = Pipeline([\n",
    "        ('vect', CountVectorizer(max_features=400, ngram_range=(1,1))),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        (\"svd\", TruncatedSVD(n_components=60)),\n",
    "        ('knn_clf', KNeighborsClassifier(n_neighbors = 5)),\n",
    "    ])\n",
    "\n",
    "knn_opt_clf.fit(x_train, y_train)\n",
    "\n",
    "predicted = knn_opt_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "02cee068",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'past': 260,\n",
       " 'two': 372,\n",
       " 'years': 397,\n",
       " 'whether': 386,\n",
       " 'united': 374,\n",
       " 'states': 346,\n",
       " 'housing': 158,\n",
       " 'would': 395,\n",
       " 'financial': 117,\n",
       " 'economy': 98,\n",
       " 'prices': 280,\n",
       " 'first': 119,\n",
       " 'however': 159,\n",
       " 'may': 217,\n",
       " 'number': 243,\n",
       " 'process': 284,\n",
       " 'home': 156,\n",
       " 'income': 165,\n",
       " 'families': 113,\n",
       " 'five': 120,\n",
       " 'several': 333,\n",
       " 'high': 153,\n",
       " 'new': 238,\n",
       " 'york': 399,\n",
       " 'city': 55,\n",
       " 'washington': 382,\n",
       " '000': 0,\n",
       " 'according': 4,\n",
       " 'opportunity': 250,\n",
       " 'national': 231,\n",
       " 'less': 200,\n",
       " 'percent': 264,\n",
       " 'access': 3,\n",
       " 'house': 157,\n",
       " 'areas': 24,\n",
       " 'much': 228,\n",
       " 'media': 219,\n",
       " 'nation': 230,\n",
       " 'real': 294,\n",
       " 'data': 81,\n",
       " 'country': 74,\n",
       " 'still': 347,\n",
       " 'american': 17,\n",
       " 'second': 322,\n",
       " 'use': 376,\n",
       " 'development': 94,\n",
       " 'impact': 161,\n",
       " 'growth': 149,\n",
       " 'people': 262,\n",
       " 'market': 216,\n",
       " 'report': 304,\n",
       " 'problem': 282,\n",
       " 'could': 72,\n",
       " 'public': 288,\n",
       " 'policy': 271,\n",
       " 'future': 135,\n",
       " 'create': 78,\n",
       " 'low': 208,\n",
       " 'back': 28,\n",
       " 'although': 14,\n",
       " 'cost': 70,\n",
       " 'local': 206,\n",
       " 'increased': 167,\n",
       " 'government': 143,\n",
       " 'federal': 116,\n",
       " 'state': 345,\n",
       " 'programs': 286,\n",
       " 'means': 218,\n",
       " 'result': 308,\n",
       " 'across': 5,\n",
       " 'costs': 71,\n",
       " 'policies': 270,\n",
       " 'resources': 306,\n",
       " 'labor': 187,\n",
       " 'workers': 392,\n",
       " 'higher': 154,\n",
       " 'also': 13,\n",
       " 'global': 139,\n",
       " 'recently': 298,\n",
       " 'significant': 334,\n",
       " 'population': 275,\n",
       " 'major': 211,\n",
       " 'often': 247,\n",
       " 'continue': 68,\n",
       " 'political': 272,\n",
       " 'economic': 97,\n",
       " 'america': 16,\n",
       " 'rates': 292,\n",
       " 'changes': 50,\n",
       " 'four': 129,\n",
       " 'history': 155,\n",
       " 'early': 96,\n",
       " 'european': 107,\n",
       " 'europe': 106,\n",
       " 'center': 48,\n",
       " 'civil': 56,\n",
       " 'war': 381,\n",
       " 'south': 342,\n",
       " 'see': 326,\n",
       " 'movement': 227,\n",
       " 'many': 215,\n",
       " 'north': 241,\n",
       " 'better': 34,\n",
       " 'jobs': 183,\n",
       " 'within': 388,\n",
       " 'americans': 18,\n",
       " 'decades': 86,\n",
       " 'world': 394,\n",
       " 'another': 21,\n",
       " 'take': 354,\n",
       " 'lower': 209,\n",
       " 'change': 49,\n",
       " 'congress': 66,\n",
       " 'services': 331,\n",
       " 'likely': 204,\n",
       " 'response': 307,\n",
       " 'among': 19,\n",
       " 'majority': 212,\n",
       " 'free': 130,\n",
       " 'oil': 248,\n",
       " 'later': 190,\n",
       " 'secretary': 323,\n",
       " 'led': 196,\n",
       " 'power': 278,\n",
       " 'supreme': 352,\n",
       " 'court': 76,\n",
       " 'common': 62,\n",
       " 'rights': 310,\n",
       " 'health': 150,\n",
       " 'issues': 181,\n",
       " 'example': 111,\n",
       " 'and': 20,\n",
       " 'middle': 222,\n",
       " 'anti': 22,\n",
       " 'using': 378,\n",
       " 'education': 99,\n",
       " 'tax': 355,\n",
       " 'social': 339,\n",
       " 'called': 42,\n",
       " 'one': 249,\n",
       " 'per': 263,\n",
       " 'potential': 277,\n",
       " 'set': 332,\n",
       " 'making': 214,\n",
       " 'community': 63,\n",
       " 'society': 340,\n",
       " 'course': 75,\n",
       " 'increase': 166,\n",
       " 'include': 163,\n",
       " 'form': 125,\n",
       " 'countries': 73,\n",
       " 'recent': 297,\n",
       " 'force': 122,\n",
       " 'place': 265,\n",
       " 'benefit': 31,\n",
       " 'must': 229,\n",
       " 'order': 251,\n",
       " 'provide': 287,\n",
       " 'instead': 172,\n",
       " 'private': 281,\n",
       " 'authority': 26,\n",
       " 'way': 383,\n",
       " 'least': 195,\n",
       " 'even': 108,\n",
       " 'though': 361,\n",
       " 'average': 27,\n",
       " 'nearly': 233,\n",
       " 'small': 338,\n",
       " 'best': 33,\n",
       " 'large': 188,\n",
       " 'times': 366,\n",
       " 'next': 239,\n",
       " 'laws': 192,\n",
       " 'able': 2,\n",
       " 'full': 132,\n",
       " 'similar': 335,\n",
       " 'used': 377,\n",
       " 'terms': 358,\n",
       " 'region': 302,\n",
       " 'poor': 274,\n",
       " 'strong': 349,\n",
       " 'year': 396,\n",
       " 'total': 368,\n",
       " 'plan': 266,\n",
       " 'since': 337,\n",
       " 'billion': 38,\n",
       " 'found': 127,\n",
       " 'others': 254,\n",
       " 'cases': 47,\n",
       " '19': 1,\n",
       " 'family': 114,\n",
       " 'end': 102,\n",
       " 'greater': 146,\n",
       " 'spending': 344,\n",
       " 'benefits': 32,\n",
       " 'three': 363,\n",
       " 'rate': 291,\n",
       " 'school': 320,\n",
       " 'great': 145,\n",
       " 'find': 118,\n",
       " 'work': 391,\n",
       " 'life': 202,\n",
       " 'individuals': 169,\n",
       " 'part': 256,\n",
       " 'general': 136,\n",
       " 'black': 39,\n",
       " 'white': 387,\n",
       " 'yet': 398,\n",
       " 'every': 110,\n",
       " 'business': 41,\n",
       " 'today': 367,\n",
       " 'given': 138,\n",
       " 'current': 80,\n",
       " 'level': 201,\n",
       " 'program': 285,\n",
       " 'pay': 261,\n",
       " 'time': 365,\n",
       " 'problems': 283,\n",
       " 'become': 30,\n",
       " 'including': 164,\n",
       " 'important': 162,\n",
       " 'foreign': 124,\n",
       " 'million': 225,\n",
       " 'especially': 105,\n",
       " 'long': 207,\n",
       " 'made': 210,\n",
       " 'possible': 276,\n",
       " 'might': 223,\n",
       " 'little': 205,\n",
       " 'group': 147,\n",
       " 'thus': 364,\n",
       " 'groups': 148,\n",
       " 'working': 393,\n",
       " 'decision': 87,\n",
       " 'budget': 40,\n",
       " 'based': 29,\n",
       " 'almost': 11,\n",
       " 'de': 83,\n",
       " 'without': 389,\n",
       " 'put': 289,\n",
       " 'companies': 64,\n",
       " 'money': 226,\n",
       " 'despite': 93,\n",
       " 'things': 359,\n",
       " 'care': 45,\n",
       " 'individual': 168,\n",
       " 'trade': 370,\n",
       " 'job': 182,\n",
       " 'research': 305,\n",
       " 'going': 141,\n",
       " 'case': 46,\n",
       " 'union': 373,\n",
       " 'rules': 314,\n",
       " 'day': 82,\n",
       " 'like': 203,\n",
       " 'interest': 175,\n",
       " 'make': 213,\n",
       " 'efforts': 100,\n",
       " 'need': 235,\n",
       " 'heritage': 152,\n",
       " 'foundation': 128,\n",
       " 'key': 185,\n",
       " 'politics': 273,\n",
       " 'organization': 252,\n",
       " 'risk': 311,\n",
       " 'members': 221,\n",
       " 'party': 259,\n",
       " 'children': 51,\n",
       " 'democratic': 90,\n",
       " 'always': 15,\n",
       " 'class': 57,\n",
       " 'self': 327,\n",
       " 'legal': 198,\n",
       " 'left': 197,\n",
       " 'never': 237,\n",
       " 'sense': 329,\n",
       " 'welfare': 384,\n",
       " 'russian': 316,\n",
       " 'russia': 315,\n",
       " 'interests': 176,\n",
       " 'strategy': 348,\n",
       " 'rather': 293,\n",
       " 'well': 385,\n",
       " 'right': 309,\n",
       " 'non': 240,\n",
       " 'control': 69,\n",
       " 'simply': 336,\n",
       " 'point': 268,\n",
       " 'get': 137,\n",
       " 'human': 160,\n",
       " 'issue': 180,\n",
       " 'fact': 112,\n",
       " 'role': 312,\n",
       " 'leaders': 193,\n",
       " 'rule': 313,\n",
       " 'religious': 303,\n",
       " 'far': 115,\n",
       " 'department': 92,\n",
       " 'governments': 144,\n",
       " 'crisis': 79,\n",
       " 'plans': 267,\n",
       " 'particularly': 257,\n",
       " 'already': 12,\n",
       " 'forces': 123,\n",
       " 'come': 60,\n",
       " 'support': 351,\n",
       " 'want': 380,\n",
       " 'system': 353,\n",
       " 'us': 375,\n",
       " 'nations': 232,\n",
       " 'agreement': 10,\n",
       " 'industry': 170,\n",
       " 'defense': 88,\n",
       " 'think': 360,\n",
       " 'international': 177,\n",
       " 'vote': 379,\n",
       " 'committee': 61,\n",
       " 'something': 341,\n",
       " 'administration': 9,\n",
       " 'military': 224,\n",
       " 'justice': 184,\n",
       " 'different': 95,\n",
       " 'approach': 23,\n",
       " 'clear': 58,\n",
       " 'parties': 258,\n",
       " 'insurance': 174,\n",
       " 'good': 142,\n",
       " 'go': 140,\n",
       " 'women': 390,\n",
       " 'leadership': 194,\n",
       " 'soviet': 343,\n",
       " 'deal': 84,\n",
       " 'around': 25,\n",
       " 'funds': 134,\n",
       " 'institutions': 173,\n",
       " 'legislation': 199,\n",
       " 'organizations': 253,\n",
       " 'act': 6,\n",
       " 'actually': 8,\n",
       " 'last': 189,\n",
       " 'democracy': 89,\n",
       " 'term': 357,\n",
       " 'question': 290,\n",
       " 'students': 350,\n",
       " 'former': 126,\n",
       " 'trump': 371,\n",
       " 'said': 317,\n",
       " 'schools': 321,\n",
       " 'enough': 104,\n",
       " 'law': 191,\n",
       " 'citizens': 54,\n",
       " 'covid': 77,\n",
       " 'pandemic': 255,\n",
       " 'help': 151,\n",
       " 'big': 36,\n",
       " 'office': 245,\n",
       " 'say': 319,\n",
       " 'regime': 301,\n",
       " 'capital': 44,\n",
       " 'freedom': 131,\n",
       " 'democrats': 91,\n",
       " 'service': 330,\n",
       " 'campaign': 43,\n",
       " 'toward': 369,\n",
       " 'election': 101,\n",
       " 'constitution': 67,\n",
       " 'action': 7,\n",
       " 'reform': 300,\n",
       " 'ever': 109,\n",
       " 'china': 52,\n",
       " 'nuclear': 242,\n",
       " 'information': 171,\n",
       " 'chinese': 53,\n",
       " 'threat': 362,\n",
       " 'needs': 236,\n",
       " 'biden': 35,\n",
       " 'investment': 178,\n",
       " 'company': 65,\n",
       " 'security': 325,\n",
       " 'police': 269,\n",
       " 'sanders': 318,\n",
       " 'know': 186,\n",
       " 'president': 279,\n",
       " 'bill': 37,\n",
       " 'obama': 244,\n",
       " 'sector': 324,\n",
       " 'food': 121,\n",
       " 'necessary': 234,\n",
       " 'reason': 296,\n",
       " 'really': 295,\n",
       " 'debt': 85,\n",
       " 'senate': 328,\n",
       " 'funding': 133,\n",
       " 'officials': 246,\n",
       " 'iran': 179,\n",
       " 'taxes': 356,\n",
       " 'energy': 103,\n",
       " 'climate': 59,\n",
       " 'medicare': 220,\n",
       " 'ref': 299}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_opt_clf[0].vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "cc2890ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.01001954, 0.02633636, 0.02578821, 0.02071068, 0.01865065,\n",
       "       0.01544238, 0.01424899, 0.01390376, 0.01327717, 0.01311909,\n",
       "       0.01171396, 0.01128584, 0.01031973, 0.0101792 , 0.00910659,\n",
       "       0.00875136, 0.00846611, 0.00826602, 0.00788312, 0.00771776,\n",
       "       0.00740674, 0.00697513, 0.00691148, 0.00673303, 0.00657712,\n",
       "       0.0064485 , 0.00618466, 0.00614202, 0.00590764, 0.00582941,\n",
       "       0.00555531, 0.005446  , 0.00532617, 0.00524535, 0.00500309,\n",
       "       0.00496463, 0.00492392, 0.00473881, 0.00468373, 0.00459165,\n",
       "       0.00451527, 0.00436548, 0.00425258, 0.00417394, 0.00411131,\n",
       "       0.00405586, 0.00401627, 0.00389464, 0.00386563, 0.00384886,\n",
       "       0.00377784, 0.00369077, 0.00360974, 0.00360824, 0.00351653,\n",
       "       0.00346191, 0.00343374, 0.00337285, 0.00329714, 0.0032845 ])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_opt_clf[2].explained_variance_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9c19d6dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_vect__ngram_range</th>\n",
       "      <th>param_vect__max_features</th>\n",
       "      <th>param_svd__n_components</th>\n",
       "      <th>param_forest_clf__max_features</th>\n",
       "      <th>param_forest_clf__max_depth</th>\n",
       "      <th>param_forest_clf__criterion</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44.538229</td>\n",
       "      <td>0.430227</td>\n",
       "      <td>3.097426</td>\n",
       "      <td>0.044942</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>log2</td>\n",
       "      <td>15</td>\n",
       "      <td>entropy</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.874749</td>\n",
       "      <td>0.860959</td>\n",
       "      <td>0.860959</td>\n",
       "      <td>0.851479</td>\n",
       "      <td>0.871876</td>\n",
       "      <td>0.864005</td>\n",
       "      <td>0.008400</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11.257121</td>\n",
       "      <td>0.414312</td>\n",
       "      <td>1.563059</td>\n",
       "      <td>0.056807</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>log2</td>\n",
       "      <td>15</td>\n",
       "      <td>gini</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.873025</td>\n",
       "      <td>0.863258</td>\n",
       "      <td>0.859523</td>\n",
       "      <td>0.851192</td>\n",
       "      <td>0.868716</td>\n",
       "      <td>0.863143</td>\n",
       "      <td>0.007545</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>44.256173</td>\n",
       "      <td>0.485781</td>\n",
       "      <td>3.049271</td>\n",
       "      <td>0.055533</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>60</td>\n",
       "      <td>log2</td>\n",
       "      <td>35</td>\n",
       "      <td>entropy</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.871014</td>\n",
       "      <td>0.859810</td>\n",
       "      <td>0.862396</td>\n",
       "      <td>0.856650</td>\n",
       "      <td>0.879920</td>\n",
       "      <td>0.865958</td>\n",
       "      <td>0.008457</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>43.861682</td>\n",
       "      <td>0.323839</td>\n",
       "      <td>3.080161</td>\n",
       "      <td>0.093970</td>\n",
       "      <td>(1, 2)</td>\n",
       "      <td>400</td>\n",
       "      <td>30</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>25</td>\n",
       "      <td>entropy</td>\n",
       "      <td>{'vect__ngram_range': (1, 2), 'vect__max_featu...</td>\n",
       "      <td>0.869578</td>\n",
       "      <td>0.859236</td>\n",
       "      <td>0.862683</td>\n",
       "      <td>0.856076</td>\n",
       "      <td>0.872738</td>\n",
       "      <td>0.864062</td>\n",
       "      <td>0.006240</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>12.100364</td>\n",
       "      <td>0.267698</td>\n",
       "      <td>1.614859</td>\n",
       "      <td>0.105144</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>400</td>\n",
       "      <td>45</td>\n",
       "      <td>sqrt</td>\n",
       "      <td>15</td>\n",
       "      <td>gini</td>\n",
       "      <td>{'vect__ngram_range': (1, 1), 'vect__max_featu...</td>\n",
       "      <td>0.870152</td>\n",
       "      <td>0.860098</td>\n",
       "      <td>0.855214</td>\n",
       "      <td>0.856938</td>\n",
       "      <td>0.874174</td>\n",
       "      <td>0.863315</td>\n",
       "      <td>0.007501</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "1       44.538229      0.430227         3.097426        0.044942   \n",
       "11      11.257121      0.414312         1.563059        0.056807   \n",
       "25      44.256173      0.485781         3.049271        0.055533   \n",
       "29      43.861682      0.323839         3.080161        0.093970   \n",
       "32      12.100364      0.267698         1.614859        0.105144   \n",
       "\n",
       "   param_vect__ngram_range param_vect__max_features param_svd__n_components  \\\n",
       "1                   (1, 2)                      400                      60   \n",
       "11                  (1, 1)                      400                      60   \n",
       "25                  (1, 2)                      400                      60   \n",
       "29                  (1, 2)                      400                      30   \n",
       "32                  (1, 1)                      400                      45   \n",
       "\n",
       "   param_forest_clf__max_features param_forest_clf__max_depth  \\\n",
       "1                            log2                          15   \n",
       "11                           log2                          15   \n",
       "25                           log2                          35   \n",
       "29                           sqrt                          25   \n",
       "32                           sqrt                          15   \n",
       "\n",
       "   param_forest_clf__criterion  \\\n",
       "1                      entropy   \n",
       "11                        gini   \n",
       "25                     entropy   \n",
       "29                     entropy   \n",
       "32                        gini   \n",
       "\n",
       "                                               params  split0_test_score  \\\n",
       "1   {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.874749   \n",
       "11  {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.873025   \n",
       "25  {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.871014   \n",
       "29  {'vect__ngram_range': (1, 2), 'vect__max_featu...           0.869578   \n",
       "32  {'vect__ngram_range': (1, 1), 'vect__max_featu...           0.870152   \n",
       "\n",
       "    split1_test_score  split2_test_score  split3_test_score  \\\n",
       "1            0.860959           0.860959           0.851479   \n",
       "11           0.863258           0.859523           0.851192   \n",
       "25           0.859810           0.862396           0.856650   \n",
       "29           0.859236           0.862683           0.856076   \n",
       "32           0.860098           0.855214           0.856938   \n",
       "\n",
       "    split4_test_score  mean_test_score  std_test_score  rank_test_score  \n",
       "1            0.871876         0.864005        0.008400                3  \n",
       "11           0.868716         0.863143        0.007545                5  \n",
       "25           0.879920         0.865958        0.008457                1  \n",
       "29           0.872738         0.864062        0.006240                2  \n",
       "32           0.874174         0.863315        0.007501                4  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_cv_results = bi.blosc_read(\"./data/forest_grid_search_result.dat\")\n",
    "forest_top5 = forest_cv_results.query(\"rank_test_score <= 5\")\n",
    "forest_top5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8526e717",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.54      0.54      0.54      2097\n",
      "  right-wing       0.57      0.58      0.58      2255\n",
      "\n",
      "    accuracy                           0.56      4352\n",
      "   macro avg       0.56      0.56      0.56      4352\n",
      "weighted avg       0.56      0.56      0.56      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1135,  962],\n",
       "       [ 955, 1300]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_clf = Pipeline([\n",
    "        ('vect', CountVectorizer()),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        (\"svd\", TruncatedSVD()),\n",
    "        ('forest_clf', RandomForestClassifier()),\n",
    "    ])\n",
    "\n",
    "forest_clf.fit(x_train, y_train)\n",
    "\n",
    "predicted = forest_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e2ecbfab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "   left-wing       0.89      0.86      0.87      2097\n",
      "  right-wing       0.87      0.90      0.88      2255\n",
      "\n",
      "    accuracy                           0.88      4352\n",
      "   macro avg       0.88      0.88      0.88      4352\n",
      "weighted avg       0.88      0.88      0.88      4352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[1794,  303],\n",
       "       [ 225, 2030]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_opt_clf = Pipeline([\n",
    "        ('vect', CountVectorizer(max_features=400, ngram_range=(1,2))),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        (\"svd\", TruncatedSVD(n_components=60)),\n",
    "        ('forest_clf', RandomForestClassifier(max_features=\"log2\",max_depth=35,\n",
    "        criterion=\"entropy\")), \n",
    "    ])\n",
    "\n",
    "forest_opt_clf.fit(x_train, y_train)\n",
    "\n",
    "predicted = forest_opt_clf.predict(x_test)\n",
    "np.mean(predicted == y_test)\n",
    "\n",
    "print(metrics.classification_report(y_test, predicted))\n",
    "\n",
    "metrics.confusion_matrix(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d544d6e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'past': 261,\n",
       " 'two': 371,\n",
       " 'years': 397,\n",
       " 'whether': 386,\n",
       " 'united': 373,\n",
       " 'states': 345,\n",
       " 'housing': 158,\n",
       " 'would': 395,\n",
       " 'financial': 116,\n",
       " 'economy': 98,\n",
       " 'prices': 281,\n",
       " 'first': 118,\n",
       " 'however': 159,\n",
       " 'may': 217,\n",
       " 'number': 244,\n",
       " 'process': 285,\n",
       " 'home': 156,\n",
       " 'income': 165,\n",
       " 'families': 112,\n",
       " 'five': 119,\n",
       " 'several': 332,\n",
       " 'high': 153,\n",
       " 'new': 238,\n",
       " 'york': 399,\n",
       " 'city': 54,\n",
       " 'washington': 382,\n",
       " '000': 0,\n",
       " 'according': 4,\n",
       " 'opportunity': 251,\n",
       " 'national': 231,\n",
       " 'less': 200,\n",
       " 'percent': 265,\n",
       " 'access': 3,\n",
       " 'house': 157,\n",
       " 'areas': 24,\n",
       " 'much': 228,\n",
       " 'media': 219,\n",
       " 'nation': 230,\n",
       " 'real': 295,\n",
       " 'data': 81,\n",
       " 'country': 73,\n",
       " 'still': 346,\n",
       " 'american': 17,\n",
       " 'second': 322,\n",
       " 'use': 376,\n",
       " 'development': 94,\n",
       " 'impact': 161,\n",
       " 'growth': 147,\n",
       " 'people': 263,\n",
       " 'market': 216,\n",
       " 'report': 304,\n",
       " 'problem': 283,\n",
       " 'could': 71,\n",
       " 'public': 289,\n",
       " 'policy': 272,\n",
       " 'future': 134,\n",
       " 'create': 78,\n",
       " 'low': 208,\n",
       " 'back': 28,\n",
       " 'although': 14,\n",
       " 'cost': 69,\n",
       " 'local': 206,\n",
       " 'increased': 167,\n",
       " 'government': 142,\n",
       " 'federal': 115,\n",
       " 'state': 344,\n",
       " 'programs': 287,\n",
       " 'means': 218,\n",
       " 'result': 308,\n",
       " 'across': 5,\n",
       " 'costs': 70,\n",
       " 'policies': 271,\n",
       " 'resources': 306,\n",
       " 'labor': 187,\n",
       " 'workers': 392,\n",
       " 'higher': 154,\n",
       " 'also': 13,\n",
       " 'global': 138,\n",
       " 'significant': 333,\n",
       " 'population': 276,\n",
       " 'major': 211,\n",
       " 'often': 248,\n",
       " 'continue': 67,\n",
       " 'political': 273,\n",
       " 'economic': 97,\n",
       " 'america': 16,\n",
       " 'rates': 293,\n",
       " 'changes': 49,\n",
       " 'four': 128,\n",
       " 'history': 155,\n",
       " 'early': 96,\n",
       " 'european': 107,\n",
       " 'europe': 106,\n",
       " 'center': 47,\n",
       " 'civil': 55,\n",
       " 'war': 381,\n",
       " 'south': 341,\n",
       " 'see': 325,\n",
       " 'movement': 227,\n",
       " 'many': 215,\n",
       " 'north': 242,\n",
       " 'better': 33,\n",
       " 'jobs': 183,\n",
       " 'within': 388,\n",
       " 'americans': 18,\n",
       " 'decades': 86,\n",
       " 'world': 394,\n",
       " 'another': 21,\n",
       " 'take': 353,\n",
       " 'lower': 209,\n",
       " 'change': 48,\n",
       " 'congress': 65,\n",
       " 'services': 330,\n",
       " 'likely': 204,\n",
       " 'response': 307,\n",
       " 'among': 19,\n",
       " 'majority': 212,\n",
       " 'free': 129,\n",
       " 'oil': 249,\n",
       " 'later': 190,\n",
       " 'led': 196,\n",
       " 'power': 279,\n",
       " 'supreme': 350,\n",
       " 'court': 75,\n",
       " 'common': 61,\n",
       " 'rights': 310,\n",
       " 'health': 148,\n",
       " 'issues': 181,\n",
       " 'example': 110,\n",
       " 'and': 20,\n",
       " 'middle': 222,\n",
       " 'anti': 22,\n",
       " 'using': 378,\n",
       " 'education': 99,\n",
       " 'tax': 354,\n",
       " 'social': 338,\n",
       " 'called': 41,\n",
       " 'one': 250,\n",
       " 'per': 264,\n",
       " 'potential': 278,\n",
       " 'set': 331,\n",
       " 'making': 214,\n",
       " 'community': 62,\n",
       " 'society': 339,\n",
       " 'course': 74,\n",
       " 'increase': 166,\n",
       " 'include': 163,\n",
       " 'form': 124,\n",
       " 'countries': 72,\n",
       " 'recent': 298,\n",
       " 'force': 121,\n",
       " 'place': 266,\n",
       " 'must': 229,\n",
       " 'order': 252,\n",
       " 'provide': 288,\n",
       " 'instead': 172,\n",
       " 'private': 282,\n",
       " 'authority': 26,\n",
       " 'way': 383,\n",
       " 'least': 195,\n",
       " 'even': 108,\n",
       " 'though': 360,\n",
       " 'average': 27,\n",
       " 'nearly': 233,\n",
       " 'small': 337,\n",
       " 'best': 32,\n",
       " 'large': 188,\n",
       " 'times': 365,\n",
       " 'next': 240,\n",
       " 'laws': 192,\n",
       " 'able': 2,\n",
       " 'full': 131,\n",
       " 'similar': 334,\n",
       " 'used': 377,\n",
       " 'terms': 357,\n",
       " 'region': 302,\n",
       " 'poor': 275,\n",
       " 'year': 396,\n",
       " 'total': 367,\n",
       " 'plan': 267,\n",
       " 'since': 336,\n",
       " 'billion': 37,\n",
       " 'found': 126,\n",
       " 'others': 255,\n",
       " 'cases': 46,\n",
       " '19': 1,\n",
       " 'family': 113,\n",
       " 'end': 102,\n",
       " 'spending': 343,\n",
       " 'benefits': 31,\n",
       " 'three': 362,\n",
       " 'rate': 292,\n",
       " 'school': 320,\n",
       " 'great': 144,\n",
       " 'find': 117,\n",
       " 'work': 391,\n",
       " 'life': 202,\n",
       " 'individuals': 169,\n",
       " 'part': 257,\n",
       " 'general': 135,\n",
       " 'black': 38,\n",
       " 'white': 387,\n",
       " 'yet': 398,\n",
       " 'every': 109,\n",
       " 'business': 40,\n",
       " 'today': 366,\n",
       " 'given': 137,\n",
       " 'current': 80,\n",
       " 'level': 201,\n",
       " 'program': 286,\n",
       " 'pay': 262,\n",
       " 'time': 364,\n",
       " 'problems': 284,\n",
       " 'become': 30,\n",
       " 'including': 164,\n",
       " 'important': 162,\n",
       " 'foreign': 123,\n",
       " 'million': 225,\n",
       " 'especially': 105,\n",
       " 'long': 207,\n",
       " 'made': 210,\n",
       " 'possible': 277,\n",
       " 'might': 223,\n",
       " 'little': 205,\n",
       " 'group': 145,\n",
       " 'thus': 363,\n",
       " 'groups': 146,\n",
       " 'working': 393,\n",
       " 'decision': 87,\n",
       " 'budget': 39,\n",
       " 'based': 29,\n",
       " 'almost': 11,\n",
       " 'de': 83,\n",
       " 'without': 389,\n",
       " 'put': 290,\n",
       " 'companies': 63,\n",
       " 'money': 226,\n",
       " 'despite': 93,\n",
       " 'things': 358,\n",
       " 'care': 44,\n",
       " 'individual': 168,\n",
       " 'trade': 369,\n",
       " 'job': 182,\n",
       " 'research': 305,\n",
       " 'going': 140,\n",
       " 'case': 45,\n",
       " 'union': 372,\n",
       " 'rules': 314,\n",
       " 'day': 82,\n",
       " 'like': 203,\n",
       " 'interest': 175,\n",
       " 'make': 213,\n",
       " 'efforts': 100,\n",
       " 'need': 235,\n",
       " 'heritage': 151,\n",
       " 'foundation': 127,\n",
       " 'united states': 374,\n",
       " 'new york': 239,\n",
       " 'supreme court': 351,\n",
       " 'heritage foundation': 152,\n",
       " 'key': 185,\n",
       " 'politics': 274,\n",
       " 'organization': 253,\n",
       " 'risk': 311,\n",
       " 'members': 221,\n",
       " 'party': 260,\n",
       " 'children': 50,\n",
       " 'democratic': 90,\n",
       " 'always': 15,\n",
       " 'class': 56,\n",
       " 'self': 326,\n",
       " 'legal': 198,\n",
       " 'left': 197,\n",
       " 'never': 237,\n",
       " 'sense': 328,\n",
       " 'welfare': 384,\n",
       " 'russian': 316,\n",
       " 'russia': 315,\n",
       " 'interests': 176,\n",
       " 'strategy': 347,\n",
       " 'rather': 294,\n",
       " 'well': 385,\n",
       " 'right': 309,\n",
       " 'non': 241,\n",
       " 'control': 68,\n",
       " 'simply': 335,\n",
       " 'point': 269,\n",
       " 'get': 136,\n",
       " 'human': 160,\n",
       " 'issue': 180,\n",
       " 'fact': 111,\n",
       " 'role': 312,\n",
       " 'leaders': 193,\n",
       " 'rule': 313,\n",
       " 'religious': 303,\n",
       " 'far': 114,\n",
       " 'department': 92,\n",
       " 'governments': 143,\n",
       " 'crisis': 79,\n",
       " 'plans': 268,\n",
       " 'particularly': 258,\n",
       " 'already': 12,\n",
       " 'forces': 122,\n",
       " 'come': 59,\n",
       " 'support': 349,\n",
       " 'want': 380,\n",
       " 'system': 352,\n",
       " 'us': 375,\n",
       " 'nations': 232,\n",
       " 'agreement': 10,\n",
       " 'industry': 170,\n",
       " 'defense': 88,\n",
       " 'think': 359,\n",
       " 'international': 177,\n",
       " 'vote': 379,\n",
       " 'committee': 60,\n",
       " 'something': 340,\n",
       " 'administration': 9,\n",
       " 'military': 224,\n",
       " 'justice': 184,\n",
       " 'different': 95,\n",
       " 'approach': 23,\n",
       " 'clear': 57,\n",
       " 'parties': 259,\n",
       " 'insurance': 174,\n",
       " 'good': 141,\n",
       " 'go': 139,\n",
       " 'women': 390,\n",
       " 'leadership': 194,\n",
       " 'soviet': 342,\n",
       " 'deal': 84,\n",
       " 'around': 25,\n",
       " 'funds': 133,\n",
       " 'institutions': 173,\n",
       " 'legislation': 199,\n",
       " 'organizations': 254,\n",
       " 'act': 6,\n",
       " 'actually': 8,\n",
       " 'last': 189,\n",
       " 'democracy': 89,\n",
       " 'term': 356,\n",
       " 'question': 291,\n",
       " 'students': 348,\n",
       " 'former': 125,\n",
       " 'trump': 370,\n",
       " 'said': 317,\n",
       " 'schools': 321,\n",
       " 'enough': 104,\n",
       " 'law': 191,\n",
       " 'citizens': 53,\n",
       " 'covid': 76,\n",
       " 'pandemic': 256,\n",
       " 'help': 150,\n",
       " 'big': 35,\n",
       " 'office': 246,\n",
       " 'covid 19': 77,\n",
       " 'say': 319,\n",
       " 'regime': 301,\n",
       " 'capital': 43,\n",
       " 'freedom': 130,\n",
       " 'democrats': 91,\n",
       " 'service': 329,\n",
       " 'campaign': 42,\n",
       " 'toward': 368,\n",
       " 'election': 101,\n",
       " 'constitution': 66,\n",
       " 'action': 7,\n",
       " 'reform': 300,\n",
       " 'health care': 149,\n",
       " 'china': 51,\n",
       " 'nuclear': 243,\n",
       " 'information': 171,\n",
       " 'chinese': 52,\n",
       " 'threat': 361,\n",
       " 'needs': 236,\n",
       " 'biden': 34,\n",
       " 'investment': 178,\n",
       " 'company': 64,\n",
       " 'security': 324,\n",
       " 'police': 270,\n",
       " 'sanders': 318,\n",
       " 'know': 186,\n",
       " 'president': 280,\n",
       " 'bill': 36,\n",
       " 'obama': 245,\n",
       " 'sector': 323,\n",
       " 'food': 120,\n",
       " 'necessary': 234,\n",
       " 'reason': 297,\n",
       " 'really': 296,\n",
       " 'debt': 85,\n",
       " 'senate': 327,\n",
       " 'funding': 132,\n",
       " 'officials': 247,\n",
       " 'iran': 179,\n",
       " 'taxes': 355,\n",
       " 'energy': 103,\n",
       " 'climate': 58,\n",
       " 'medicare': 220,\n",
       " 'ref': 299}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_opt_clf[0].vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "220f1b7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00999342, 0.02655163, 0.02587907, 0.02104172, 0.01868057,\n",
       "       0.01556065, 0.01453942, 0.0141053 , 0.01339169, 0.01308897,\n",
       "       0.01171164, 0.01129167, 0.01045946, 0.01028872, 0.00970334,\n",
       "       0.00897714, 0.00843318, 0.0082745 , 0.00811222, 0.00785617,\n",
       "       0.00745173, 0.00715057, 0.00712232, 0.00679096, 0.00656512,\n",
       "       0.00647889, 0.00628874, 0.00617092, 0.00594727, 0.00585134,\n",
       "       0.00557567, 0.00553865, 0.00534393, 0.00523238, 0.00506958,\n",
       "       0.00498049, 0.0049511 , 0.00473881, 0.0047036 , 0.00457587,\n",
       "       0.0045227 , 0.00448466, 0.00427141, 0.00418954, 0.00412463,\n",
       "       0.00406127, 0.00403277, 0.00397749, 0.00390995, 0.00385661,\n",
       "       0.00378418, 0.00374152, 0.00365741, 0.00360852, 0.0035587 ,\n",
       "       0.00345572, 0.00344809, 0.00339067, 0.0033253 , 0.00324853])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_opt_clf[2].explained_variance_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "political_sentiment",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15 (main, Nov 24 2022, 14:31:59) \n[GCC 11.2.0]"
  },
  "vscode": {
   "interpreter": {
    "hash": "6e38ea1e5deeed82a3b294ec323014828b987cd6fe46463ab21a833d1705cc24"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}