utilites.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "import os\n",
    "import torch\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "from transformers import logging\n",
    "logging.set_verbosity_error()\n",
    "warnings.filterwarnings('ignore')\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "from transformers import AutoTokenizer, AutoModel\n",
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.8.5 (default, Sep  4 2020, 02:22:02) \\n[Clang 10.0.0 ]'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataClass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class AnyData(Dataset):\n",
    "    \n",
    "    def __init__(self):\n",
    "        # xy = np.loadtxt('', delimiter='', dtype=np.float32, skiprows=1)\n",
    "        # self.x\n",
    "        # self.y\n",
    "        # self.n_namples = xy.shape[0]\n",
    "        pass\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        # return self.x[index], self.y[index]\n",
    "        pass\n",
    "        \n",
    "    def __len__(self):\n",
    "        # return self.n_namples\n",
    "        pass\n",
    "    \n",
    "dataset = AnyData()\n",
    "# dataset[0]\n",
    "dataloader = DataLoader(dataset=dataset, batch_size=4, shuffle=True, num_workers=2)\n",
    "# dataiter = iter(dataloader)\n",
    "# data_1 = dataiter.next()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# counts = Counter(words)\n",
    "# cv = CountVectorizer(vocabulary=counts.keys())\n",
    "# cv.fit_transform(i).toarray()[0].tolist()\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer(\n",
    "    analyzer = 'word', \n",
    "    ngram_range=(1,1), \n",
    "    min_df=0.00001, \n",
    "    max_features=600, \n",
    "    max_df=0.95, \n",
    "    stop_words='english', \n",
    "    norm='l2',\n",
    "    use_idf=True\n",
    ")\n",
    "x = vectorizer.fit_transform(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### split & shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "def split_train_test(corpus, labels, ratios=False):\n",
    "    \"\"\"\n",
    "    Split data to train / test with no percentage preservation for each class sample.\n",
    "    Insted, specify class ratios in ratios arg\n",
    "    EXAMPLE\n",
    "    ratios = \n",
    "    \"\"\"\n",
    "    X_train, X_test, y_train, y_test = [], [], [], []\n",
    "    if ratios:\n",
    "        for k, v in ratios.items():\n",
    "            train_rate = v[1] / v[0]\n",
    "            corpus_current, labels_current = data_by_class_selection(corpus, labels, k)\n",
    "            x_train_1, x_test_1, y_train_1, y_test_1 = train_test_split(\n",
    "                corpus_current, \n",
    "                labels_current, \n",
    "                test_size=train_rate, \n",
    "                random_state=41)\n",
    "            X_train.extend(x_train_1)\n",
    "            X_test.extend(x_test_1)\n",
    "            y_train.extend(y_train_1)\n",
    "            y_test.extend(y_test_1)\n",
    "    else:\n",
    "        for i in dict(pd.Series(labels).value_counts()).keys():\n",
    "            corpus_current, labels_current = data_by_class_selection(corpus, labels, i)\n",
    "            train_rate = 0.3\n",
    "            x_train_1, x_test_1, y_train_1, y_test_1 = train_test_split(\n",
    "                corpus_current, \n",
    "                labels_current, \n",
    "                test_size=train_rate, \n",
    "                random_state=41)\n",
    "            X_train.extend(x_train_1)\n",
    "            X_test.extend(x_test_1)\n",
    "            y_train.extend(y_train_1)\n",
    "            y_test.extend(y_test_1)\n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "def data_by_class_selection(corpus, labels, c):\n",
    "    \"\"\"\n",
    "    Condition sampling froom lists\n",
    "    \"\"\"\n",
    "    new_corpus = []\n",
    "    new_labels = []\n",
    "    for d, l in zip(corpus, labels):\n",
    "        if l == c:\n",
    "            new_corpus.append(d)\n",
    "            new_labels.append(l)\n",
    "    return new_corpus, new_labels\n",
    "\n",
    "def shuffle_data(corpus, labels):\n",
    "    combined = list(zip(corpus, labels))\n",
    "    random.shuffle(combined)\n",
    "    corpus[:], labels[:] = zip(*combined)\n",
    "    return corpus, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymystem3 import Mystem\n",
    "import re\n",
    "stemmer = Mystem()\n",
    "def preprocess(t, stem=True):\n",
    "    x = ''.join([i for i in t.lower() if i in 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя .'])  # <<<<<<< LOWER\n",
    "    x = x.replace('.', ' ')\n",
    "    if stem:\n",
    "        x = ''.join(stemmer.lemmatize(x))\n",
    "    else:\n",
    "        pass\n",
    "    x = ' '.join([i for i in x.split() if len(i) > 2])\n",
    "    x = re.sub('\\s+', ' ', x)\n",
    "    return x.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_tesseract(x):\n",
    "    import re\n",
    "    x = re.sub(r'\\\"', ' ', x)\n",
    "    x = x.replace('\\\\n', ' ')\n",
    "    x = x.replace('\\\\', ' ')\n",
    "    x = re.sub('\\s+', ' ', x)\n",
    "    return x.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pipelines (classic ML)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "folds = 2\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('tfidf', TfidfVectorizer(analyzer = 'word', min_df=0)),#, max_df=0.9, max_features=10000)),\n",
    "    # ('LinearSVC', LinearSVC(C=1, random_state=42))\n",
    "    ('LogisticRegression', LogisticRegression(solver='liblinear', random_state=42))\n",
    "    # ('SGDClassifier', SGDClassifier(loss='hinge', alpha = 0.001))# np.linspace(0.01, 0.05, 10)\n",
    "])\n",
    "parameters = { \n",
    "    # 'tfidf__max_df': (0.9, 1), # 0.8 <- 0.95 ### 0.6 \n",
    "    # 'tfidf__min_df': (0), # 0.01 <- 0.05 ### 0.005 <- 0.01 ### 0.001\n",
    "    'tfidf__ngram_range': [(1, 2), (1, 3)]\n",
    "    # 'svm__decision_function_shape': ['ovo', 'ovr'],\n",
    "    # 'svm__C': (4, 6)\n",
    "    # 'tfidf__max_features': (10000, 32000)\n",
    "}\n",
    "c_val = StratifiedKFold(n_splits=folds)\n",
    "\n",
    "grid_search_tune = GridSearchCV(pipeline, parameters, cv=c_val, verbose=3, n_jobs=4, scoring='f1_macro')\n",
    "grid_search_tune.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameters set:\")\n",
    "print(grid_search_tune.best_estimator_.steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "\n",
    "\n",
    "class TextClassifier(BaseEstimator):\n",
    "    @staticmethod\n",
    "    def _build_model():\n",
    "        model = Pipeline([\n",
    "            ('tfidf', TfidfVectorizer(analyzer='word', min_df=0, max_df=0.6, ngram_range=(1, 3))),\n",
    "            ('RandomUnderSampler', RandomUnderSampler()),\n",
    "            ('Smote', SMOTE()),\n",
    "            ('svm', SVC(C=3, kernel='linear', decision_function_shape='ovr'))\n",
    "        ])\n",
    "\n",
    "        return model\n",
    "\n",
    "    def __init__(self):\n",
    "        self.model = TextClassifier._build_model()\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.model.fit(X, y)\n",
    "\n",
    "    def predict(self, X):\n",
    "        return self.model.predict(X)\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "        return self.model.predict_proba(X)\n",
    "\n",
    "    def save(self, path):\n",
    "        self.model.save(path)\n",
    "\n",
    "    def load(self, path):\n",
    "        return self.model.load_model(path)\n",
    "model = TextClassifier()._build_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "    ('tfidf', TfidfVectorizer(analyzer = 'word', min_df=0, max_df=0.6, ngram_range=(1, 3))),\n",
    "    ('RandomUnderSampler', RandomUnderSampler()),\n",
    "    ('Smote', SMOTE()),\n",
    "    ('svm', SVC(C=3, kernel='linear', decision_function_shape='ovr'))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "import numpy as np\n",
    "m = confusion_matrix(y_true, y_pred)\n",
    "new_m = []\n",
    "l = []\n",
    "print(m)\n",
    "for i in m:\n",
    "    s = sum(i)\n",
    "    for ii in i:\n",
    "        l.append(round(ii / s * 100))\n",
    "    new_m.append(l)\n",
    "    l = []\n",
    "print(np.array(new_m))\n",
    "\n",
    "print(f1_score(y_true, y_pred, average='macro'))\n",
    "print(accuracy_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conf matrix graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "import itertools\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "    \n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    \n",
    "    \n",
    "cnf_matrix = np.array(new_m).astype(int)\n",
    "\n",
    "    \n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "# Plot non-normalized confusion matrix\n",
    "plt.figure(num=None, figsize=(13, 11), dpi=80, facecolor='w', edgecolor='k')\n",
    "plot_confusion_matrix(cnf_matrix, classes=list(pd.Series(y_pred).value_counts().keys()),\n",
    "                      title='Confusion matrix huyatrix')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prec, rec of random classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6 0.6 0.6\n",
      "0.4 0.47619047619047616 0.43478260869565216\n",
      "0.6 0.5555555555555556 0.576923076923077\n",
      "0.512 0.5039370078740157 0.5079365079365079\n",
      "0.478 0.48975409836065575 0.4838056680161943\n",
      "0.4864 0.4846552411319251 0.48552605310441205\n",
      "0.504 0.5082011293358429 0.5060918462980318\n",
      "0.5108 0.5082587064676617 0.5095261845386534\n",
      "0.49504 0.49979807769970114 0.49740766046380774\n",
      "0.499764 0.500357423904751 0.5000605358977874\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(249882, 250118)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "for i in [10, 50, 100, 500, 1000, 5000, 7500, 10000, 50000, 1000000]:\n",
    "    preds = [np.random.randint(2) for i in range(i)]\n",
    "    y = [1] * int(i/2)\n",
    "    y.extend([0]*int(i/2))\n",
    "    y, preds = shuffle_data(y, preds)\n",
    "    tps, fns, fps, tns = 0, 0, 0, 0\n",
    "    for i in range(len(preds)):\n",
    "        if y[i] == 1 and preds[i] == 1:\n",
    "            tps += 1\n",
    "        elif y[i] == 1 and preds[i] == 0:\n",
    "            fns += 1\n",
    "        elif y[i] == 0 and preds[i] == 1:\n",
    "            fps += 1\n",
    "        elif y[i] == 0 and preds[i] == 0:\n",
    "            tns += 1\n",
    "    prec = tps / (tps + fps)\n",
    "    rec = tps / (tps + fns)\n",
    "    f1 = (2*(prec*rec))/(prec+rec)\n",
    "    print(rec, prec, f1)\n",
    "tps ,  fns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Jaccard index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def jaccard(x1, x2):\n",
    "    intersection = set(x1.split()).intersection(set(x2.split()))\n",
    "    union = set(x1.split()).union(set(x2.split()))\n",
    "    try:\n",
    "        return float(format(len(intersection) / len(union), '.2f'))\n",
    "    except ZeroDivisionError:\n",
    "        return 233.\n",
    "jaccard('Дяденька король цветной капусты хотца', 'Дяденька король цветной капусты')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# important features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVTUlEQVR4nO3dbYyd9Xnn8e9v7ZY48ZqH0sxatrVmVSst2O1uGLG0UapBZhe3QZgXQXJEgmlZWUU0pZVXxU5e5JUlqi5Ng7IgWXEWU1Acl6bCKnI3XqejaCUeCiStYxyKFbxgcCHZBorTlGTYa1+cv3cP4zNjz5x5OpPvRxqd+77uh/O/PGf8m/vhnElVIUnSv5jvAUiSFgYDQZIEGAiSpMZAkCQBBoIkqVk63wOYrksvvbTWrl074fIf/OAHvO9975u7Ac0y+1nYFlM/i6kXsJ/xnnnmme9V1c/2WjawgbB27VqefvrpCZePjo4yMjIydwOaZfazsC2mfhZTL2A/4yX5XxMt85SRJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCRjgdyprcKzd8dhZte0bxri1R30mnbj7I7O6f2mx8QhBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKk5ZyAk+WKS15N8q6v2h0m+neRvk/x5kou6lu1McjzJ80mu66pfmeRIW3ZvkrT6BUm+3OpPJlk7sy1Kks7H+RwhPABsGlc7BKyvql8E/g7YCZDkcmALcEXb5r4kS9o29wPbgHXt68w+bwO+X1U/B3wW+IPpNiNJmr5zBkJVfR34h3G1r1bVWJt9AljdpjcD+6rq7ap6ETgOXJVkJbCiqh6vqgIeBG7s2mZvm34E2Hjm6EGSNHdm4u8h/Cbw5Ta9ik5AnHGy1X7cpsfXz2zzMkBVjSV5E/gZ4HvjnyjJNjpHGQwNDTE6OjrhoE6fPj3p8kEzyP1s3zB2Vm1oWe/6TJrLf69B/v6Mt5h6AfuZir4CIcmngTHg4TOlHqvVJPXJtjm7WLUb2A0wPDxcIyMjE45tdHSUyZYPmkHup9cfwtm+YYx7jszu32c6cfPIrO6/2yB/f8ZbTL2A/UzFtO8ySrIVuB64uZ0Ggs5v/mu6VlsNvNrqq3vU37VNkqXAhYw7RSVJmn3TCoQkm4C7gBuq6p+6Fh0AtrQ7hy6jc/H4qao6BbyV5Op2feAW4NGubba26Y8CX+sKGEnSHDnnMXuSLwEjwKVJTgKfoXNX0QXAoXb994mq+q2qOppkP/AcnVNJd1TVO21Xt9O5Y2kZcLB9AewB/iTJcTpHBltmpjVJ0lScMxCq6mM9ynsmWX8XsKtH/WlgfY/6PwM3nWsckqTZ5TuVJUmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQB5xEISb6Y5PUk3+qqXZLkUJIX2uPFXct2Jjme5Pkk13XVr0xypC27N0la/YIkX271J5OsneEeJUnn4XyOEB4ANo2r7QAOV9U64HCbJ8nlwBbgirbNfUmWtG3uB7YB69rXmX3eBny/qn4O+CzwB9NtRpI0fecMhKr6OvAP48qbgb1tei9wY1d9X1W9XVUvAseBq5KsBFZU1eNVVcCD47Y5s69HgI1njh4kSXNn6TS3G6qqUwBVdSrJ+1t9FfBE13onW+3HbXp8/cw2L7d9jSV5E/gZ4HvjnzTJNjpHGQwNDTE6OjrhAE+fPj3p8kEzyP1s3zB2Vm1oWe/6TJrLf69B/v6Mt5h6AfuZiukGwkR6/WZfk9Qn2+bsYtVuYDfA8PBwjYyMTDiQ0dFRJls+aAa5n1t3PHZWbfuGMe45MtMvv3c7cfPIrO6/2yB/f8ZbTL2A/UzFdO8yeq2dBqI9vt7qJ4E1XeutBl5t9dU96u/aJslS4ELOPkUlSZpl0w2EA8DWNr0VeLSrvqXdOXQZnYvHT7XTS28lubpdH7hl3DZn9vVR4GvtOoMkaQ6d85g9yZeAEeDSJCeBzwB3A/uT3Aa8BNwEUFVHk+wHngPGgDuq6p22q9vp3LG0DDjYvgD2AH+S5DidI4MtM9KZJGlKzhkIVfWxCRZtnGD9XcCuHvWngfU96v9MCxRJ0vzxncqSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkoA+AyHJ7yU5muRbSb6U5D1JLklyKMkL7fHirvV3Jjme5Pkk13XVr0xypC27N0n6GZckaeqmHQhJVgG/AwxX1XpgCbAF2AEcrqp1wOE2T5LL2/IrgE3AfUmWtN3dD2wD1rWvTdMdlyRpevo9ZbQUWJZkKfBe4FVgM7C3Ld8L3NimNwP7qurtqnoROA5clWQlsKKqHq+qAh7s2kaSNEfS+T94mhsndwK7gB8CX62qm5O8UVUXda3z/aq6OMnngSeq6qFW3wMcBE4Ad1fVta3+YeCuqrq+x/Nto3MkwdDQ0JX79u2bcGynT59m+fLl0+5toRnkfo688uZZtaFl8NoPZ/d5N6y6cHafoMsgf3/GW0y9gP2Md8011zxTVcO9li2d7k7btYHNwGXAG8CfJvn4ZJv0qNUk9bOLVbuB3QDDw8M1MjIy4ZONjo4y2fJBM8j93LrjsbNq2zeMcc+Rab/8zsuJm0dmdf/dBvn7M95i6gXsZyr6OWV0LfBiVX23qn4MfAX4FeC1dhqI9vh6W/8ksKZr+9V0TjGdbNPj65KkOdRPILwEXJ3kve2uoI3AMeAAsLWtsxV4tE0fALYkuSDJZXQuHj9VVaeAt5Jc3fZzS9c2kqQ5Mu1j9qp6MskjwLPAGPANOqdzlgP7k9xGJzRuausfTbIfeK6tf0dVvdN2dzvwALCMznWFg9MdlyRpevo6iVtVnwE+M678Np2jhV7r76JzEXp8/WlgfT9jkST1x3cqS5IAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkoM+/qazBsXbHY/M9BEkLnEcIkiTAQJAkNX0FQpKLkjyS5NtJjiX55SSXJDmU5IX2eHHX+juTHE/yfJLruupXJjnSlt2bJP2MS5I0df0eIXwO+Muq+nngl4BjwA7gcFWtAw63eZJcDmwBrgA2AfclWdL2cz+wDVjXvjb1OS5J0hRNOxCSrAB+FdgDUFU/qqo3gM3A3rbaXuDGNr0Z2FdVb1fVi8Bx4KokK4EVVfV4VRXwYNc2kqQ5ks7/wdPYMPm3wG7gOTpHB88AdwKvVNVFXet9v6ouTvJ54ImqeqjV9wAHgRPA3VV1bat/GLirqq7v8Zzb6BxJMDQ0dOW+ffsmHN/p06dZvnz5tHpbiPrt58grb87gaPo3tAxe++HsPseGVRfO7hN0WUyvt8XUC9jPeNdcc80zVTXca1k/t50uBT4IfLKqnkzyOdrpoQn0ui5Qk9TPLlbtphNCDA8P18jIyIRPNjo6ymTLB02//dy6wG473b5hjHuOzO5dzyduHpnV/XdbTK+3xdQL2M9U9HMN4SRwsqqebPOP0AmI19ppINrj613rr+nafjXwaquv7lGXJM2haQdCVf098HKSD7TSRjqnjw4AW1ttK/Bomz4AbElyQZLL6Fw8fqqqTgFvJbm63V10S9c2kqQ50u8x+yeBh5P8NPAd4DfohMz+JLcBLwE3AVTV0ST76YTGGHBHVb3T9nM78ACwjM51hYN9jkuSNEV9BUJVfRPodXFi4wTr7wJ29ag/DazvZyySpP74TmVJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkScAMBEKSJUm+keQv2vwlSQ4leaE9Xty17s4kx5M8n+S6rvqVSY60ZfcmSb/jkiRNzUwcIdwJHOua3wEcrqp1wOE2T5LLgS3AFcAm4L4kS9o29wPbgHXta9MMjEuSNAV9BUKS1cBHgC90lTcDe9v0XuDGrvq+qnq7ql4EjgNXJVkJrKiqx6uqgAe7tpEkzZGlfW7/x8DvA/+yqzZUVacAqupUkve3+irgia71Trbaj9v0+PpZkmyjcyTB0NAQo6OjEw7s9OnTky4fNP32s33D2MwNZgYMLZv9Mc3l938xvd4WUy9gP1Mx7UBIcj3welU9k2TkfDbpUatJ6mcXq3YDuwGGh4drZGTipx0dHWWy5YOm335u3fHYzA1mBmzfMMY9R/r9fWRyJ24emdX9d1tMr7fF1AvYz1T08xP5IeCGJL8OvAdYkeQh4LUkK9vRwUrg9bb+SWBN1/argVdbfXWPuiRpDk37GkJV7ayq1VW1ls7F4q9V1ceBA8DWttpW4NE2fQDYkuSCJJfRuXj8VDu99FaSq9vdRbd0bSNJmiOzccx+N7A/yW3AS8BNAFV1NMl+4DlgDLijqt5p29wOPAAsAw62L0nSHJqRQKiqUWC0Tf9vYOME6+0CdvWoPw2sn4mxSJKmx3cqS5IAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkC+giEJGuS/FWSY0mOJrmz1S9JcijJC+3x4q5tdiY5nuT5JNd11a9McqQtuzdJ+mtLkjRV/RwhjAHbq+oXgKuBO5JcDuwADlfVOuBwm6ct2wJcAWwC7kuypO3rfmAbsK59bepjXJKkaZh2IFTVqap6tk2/BRwDVgGbgb1ttb3AjW16M7Cvqt6uqheB48BVSVYCK6rq8aoq4MGubSRJcySd/4P73EmyFvg6sB54qaou6lr2/aq6OMnngSeq6qFW3wMcBE4Ad1fVta3+YeCuqrq+x/Nso3MkwdDQ0JX79u2bcEynT59m+fLlffe2UPTbz5FX3pzB0fRvaBm89sPZfY4Nqy6c3Sfospheb4upF7Cf8a655ppnqmq417Kl095rk2Q58GfA71bVP05y+r/XgpqkfnaxajewG2B4eLhGRkYmHNfo6CiTLR80/fZz647HZm4wM2D7hjHuOdL3y29SJ24emdX9d1tMr7fF1AvYz1T0dZdRkp+iEwYPV9VXWvm1dhqI9vh6q58E1nRtvhp4tdVX96hLkuZQP3cZBdgDHKuqP+padADY2qa3Ao921bckuSDJZXQuHj9VVaeAt5Jc3fZ5S9c2kqQ50s8x+4eATwBHknyz1T4F3A3sT3Ib8BJwE0BVHU2yH3iOzh1Kd1TVO22724EHgGV0risc7GNckqRpmHYgVNX/pPf5f4CNE2yzC9jVo/40nQvSkqR54juVJUmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkS0MffVJYWurU7Hpuz59q+YYxb2/OduPsjc/a80kzyCEGSBBgIkqTGQJAkAQsoEJJsSvJ8kuNJdsz3eCTpJ82CuKicZAnwX4H/AJwE/jrJgap6bn5HNvOme6Gz+6KlJM2GBREIwFXA8ar6DkCSfcBmYNEFgha/uby7aTzvcFI/UlXzPQaSfBTYVFX/qc1/Avj3VfXb49bbBmxrsx8Anp9kt5cC35uF4c4X+1nYFlM/i6kXsJ/x/nVV/WyvBQvlCCE9amclVVXtBnaf1w6Tp6tquN+BLRT2s7Atpn4WUy9gP1OxUC4qnwTWdM2vBl6dp7FI0k+khRIIfw2sS3JZkp8GtgAH5nlMkvQTZUGcMqqqsSS/Dfx3YAnwxao62uduz+vU0gCxn4VtMfWzmHoB+zlvC+KisiRp/i2UU0aSpHlmIEiSgEUcCEn+c5JKcmlXbWf7aIznk1w3n+M7H0n+MMm3k/xtkj9PclHXsoHq5YxB/4iSJGuS/FWSY0mOJrmz1S9JcijJC+3x4vke6/lKsiTJN5L8RZsf2F4AklyU5JH2s3MsyS8Pak9Jfq+9zr6V5EtJ3jObvSzKQEiyhs7HYLzUVbuczt1LVwCbgPvaR2YsZIeA9VX1i8DfATthYHvp/oiSXwMuBz7WehkkY8D2qvoF4GrgjtbDDuBwVa0DDrf5QXEncKxrfpB7Afgc8JdV9fPAL9HpbeB6SrIK+B1guKrW07nhZguz2MuiDATgs8Dv8+43t20G9lXV21X1InCczkdmLFhV9dWqGmuzT9B5fwYMYC/N//uIkqr6EXDmI0oGRlWdqqpn2/RbdP6zWUWnj71ttb3AjfMywClKshr4CPCFrvJA9gKQZAXwq8AegKr6UVW9weD2tBRYlmQp8F4678+atV4WXSAkuQF4par+ZtyiVcDLXfMnW21Q/CZwsE0Pai+DOu6ekqwF/h3wJDBUVaegExrA++dxaFPxx3R+efo/XbVB7QXg3wDfBf5bOw32hSTvYwB7qqpXgP9C50zHKeDNqvoqs9jLgngfwlQl+R/Av+qx6NPAp4D/2GuzHrV5v+d2sl6q6tG2zqfpnKp4+MxmPdaf917Ow6CO+yxJlgN/BvxuVf1j0qu1hS3J9cDrVfVMkpF5Hs5MWQp8EPhkVT2Z5HMMwOmhXtq1gc3AZcAbwJ8m+fhsPudABkJVXdurnmQDnX+8v2k/oKuBZ5NcxQL9eIyJejkjyVbgemBj/f83jSzIXs7DoI77XZL8FJ0weLiqvtLKryVZWVWnkqwEXp+/EZ63DwE3JPl14D3AiiQPMZi9nHESOFlVT7b5R+gEwiD2dC3wYlV9FyDJV4BfYRZ7WVSnjKrqSFW9v6rWVtVaOi+OD1bV39P5KIwtSS5IchmwDnhqHod7Tkk2AXcBN1TVP3UtGrhemoH/iJJ0ftPYAxyrqj/qWnQA2NqmtwKPzvXYpqqqdlbV6vazsgX4WlV9nAHs5Yz2s/5ykg+00kY6H6M/iD29BFyd5L3tdbeRzjWrWetlII8QpqOqjibZT+fFMQbcUVXvzPOwzuXzwAXAoXbE80RV/daA9jJbH1Ey1z4EfAI4kuSbrfYp4G5gf5Lb6Pwg3zQ/w5sRg97LJ4GH2y8d3wF+g84vvwPVUzvl9QjwLJ2f82/Q+diK5cxSL350hSQJWGSnjCRJ02cgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJzf8FadZOYmaeLlQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(grid_search_tune.best_estimator_.steps[1][1].coef_[0]).hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "coefs = grid_search_tune.best_estimator_.steps[1][1].coef_\n",
    "without_bias = coefs - grid_search_tune.best_estimator_.steps[1][1].intercept_ * len(coefs)\n",
    "\n",
    "features = grid_search_tune.best_estimator_.steps[0][1].get_feature_names()\n",
    "important_features = []\n",
    "for i in range(len(features)):\n",
    "    if without_bias[0][i] >= 20.:\n",
    "        important_features.append(features[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Open"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_files(path, ext):\n",
    "    import os\n",
    "    paths_to_files = []\n",
    "    for root, dir, files in os.walk(path):\n",
    "        if files:\n",
    "            for f in files:\n",
    "                if ext in f:\n",
    "                    paths_to_files.append(os.path.join(root, f))\n",
    "    if paths_to_files:\n",
    "        return paths_to_files\n",
    "    else:\n",
    "        raise ValueError('No such directory')\n",
    "# get_files('<path>', 'csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update json\n",
    "try:\n",
    "    with open('.json') as json_file:\n",
    "        data = json.load(json_file)\n",
    "    data.update(user_data)\n",
    "    json.dump(data, open(\".json\", 'w'))\n",
    "except FileNotFoundError:\n",
    "    json.dump(user_data, open(\".json\", 'w'))\n",
    "user_data = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "new_labels1, new_corpus1 = [], []\n",
    "with open('type_march_.csv', 'rt') as csvfile:\n",
    "    readCSV = csv.reader(csvfile, delimiter=',')\n",
    "    for n, i in enumerate(readCSV):\n",
    "        # print(i)\n",
    "        # break\n",
    "        if n != 0:\n",
    "            new_labels1.append(i[0])\n",
    "            new_corpus1.append(i[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create list of items from list of lists of items\n",
    "flat_list = [item for sublist in t for item in sublist]\n",
    "\n",
    "# sort dict by values\n",
    "{k: v for k, v in sorted(result.items(), key=lambda item: item[1])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import copy\n",
    "def top_n_max(series, top=2, diraction=max):\n",
    "    series = list(set(series.tolist()))\n",
    "    if diraction == max:\n",
    "      top_result = sorted(series)[-top:]\n",
    "    else:\n",
    "      top_result = sorted(series)[:top]\n",
    "    return top_result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}