nb_svm_logreg_arup_colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "nb_svm_logreg.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hyf6bEO05BZf",
        "outputId": "aafd36b5-29d0-40a0-ec0d-dd539b40d955",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "tf.test.gpu_device_name()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/device:GPU:0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "oii90RZDk8VZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "outputId": "75dfaaf7-c3e3-4c16-add7-5654ebd56e77"
      },
      "source": [
        "# !python3 --version\n",
        "# !pip --version\n",
        "\n",
        "# References:\n",
        "# https://towardsdatascience.com/multi-label-text-classification-with-scikit-learn-30714b7819c5\n",
        "\n",
        "\n",
        "# UNCOMMENT THE BELOW IMPORTS IF YOU ARE NOT USING THE COLAB GPU OR IF YOU GET IMPORT ERRORS\n",
        "\n",
        "# !pip3 install re \n",
        "# !pip3 install matplotlib \n",
        "# !pip3 install numpy \n",
        "# !pip3 install pandas \n",
        "# !pip3 install scikit-learn \n",
        "# !pip3 install nltk \n",
        "!pip3 install seaborn\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.6/dist-packages (0.10.0)\n",
            "Requirement already satisfied: matplotlib>=2.1.2 in /usr/local/lib/python3.6/dist-packages (from seaborn) (3.2.1)\n",
            "Requirement already satisfied: scipy>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.4.1)\n",
            "Requirement already satisfied: pandas>=0.22.0 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.0.3)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.18.2)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.1.2->seaborn) (2.8.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.1.2->seaborn) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.1.2->seaborn) (2.4.6)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.1.2->seaborn) (1.1.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.22.0->seaborn) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib>=2.1.2->seaborn) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib>=2.1.2->seaborn) (46.0.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "picKhQCmkoBD",
        "outputId": "8d26f42b-fbd5-4076-d6d4-a62ee1f3b6c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "%matplotlib inline\n",
        "import re\n",
        "import matplotlib\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "print('Stopwords have finished downloading successfully')\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.metrics import accuracy_score, classification_report\n",
        "from sklearn.multiclass import OneVsRestClassifier\n",
        "from nltk.corpus import stopwords\n",
        "stop_words = set(stopwords.words('english'))\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import Pipeline\n",
        "import seaborn as sns\n",
        "print('All imports were successful')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "Stopwords have finished downloading successfully\n",
            "All imports were successful\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "l4y2pMcqoBbq",
        "outputId": "15f67fa7-4fb6-4ae7-e9e4-5e23501c57ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "# df = pd.read_csv(\"./train.csv\", encoding = \"ISO-8859-1\")\n",
        "df = pd.read_csv(\"/content/train.csv\", encoding = \"ISO-8859-1\")\n",
        "print('The first 5 elements in the train.csv are:')\n",
        "print(df.head())\n",
        "\n",
        "# Getting the number of comments in each category\n",
        "df_toxic = df.drop(['id', 'comment_text'], axis=1)\n",
        "counts = []\n",
        "categories = list(df_toxic.columns.values)\n",
        "for i in categories:\n",
        "    counts.append((i, df_toxic[i].sum()))\n",
        "df_stats = pd.DataFrame(counts, columns=['category', 'number_of_comments'])\n",
        "print(df_stats)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The first 5 elements in the train.csv are:\n",
            "                 id  ... identity_hate\n",
            "0  0000997932d777bf  ...             0\n",
            "1  000103f0d9cfb60f  ...             0\n",
            "2  000113f07ec002fd  ...             0\n",
            "3  0001b41b1c6bb37e  ...             0\n",
            "4  0001d958c54c6e35  ...             0\n",
            "\n",
            "[5 rows x 8 columns]\n",
            "        category  number_of_comments\n",
            "0          toxic               15294\n",
            "1   severe_toxic                1595\n",
            "2        obscene                8449\n",
            "3         threat                 478\n",
            "4         insult                7877\n",
            "5  identity_hate                1405\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uKGU0e7RsD0h",
        "outputId": "84511d08-c933-4b48-a68b-cfac89b4bd26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        }
      },
      "source": [
        "# Getting a histogram of the categories\n",
        "my_colors = ['b', 'r', 'g', 'y', 'k', 'm']\n",
        "df_stats.plot(x='category', y='number_of_comments', kind='bar', legend=False, grid=True, figsize=(8, 5), color=my_colors)\n",
        "\n",
        "plt.title(\"Number of comments per category\")\n",
        "plt.ylabel('# of Occurrences', fontsize=12)\n",
        "plt.xlabel('Category', fontsize=12)\n",
        "# plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'Category')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAGHCAYAAADPzkp9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZwcVb3//9ebBMISMGxGYCJh14ig\nJLIo6gwoBAHD9YKACxGR3KsouEQ2r4IsX0XHi/JDEBQkuBAWUXKRRQRGRCDAsO+EAGZCkCUEMoBA\n4PP745wJnWFmUsn0THX3vJ+PRz+661TV6U/X9HR/+tQ5pxQRmJmZ2dC2QtkBmJmZWfmcEJiZmZkT\nAjMzM3NCYGZmZjghMDMzM5wQmJmZGU4IzPpN0jmSTijpuSXp15Kek3RzGTGYWWNwQmANR9Jjkp6S\ntFpF2ZcktZUY1kDZEfg40BQR25YdzECTNFZSSBpedizVIqlZUkfZcZg5IbBGNQw4rOwglpWkYcu4\ny4bAYxHx4kDEYz1rlISkUV6HVYcTAmtUPwamShrVfUVPvzIltUn6Un78BUn/kHSypAWSZkv6YC6f\nk1sfJnerdh1JV0laKOlvkjasqPtded18SQ9K+nTFunMknS7pMkkvAi09xLu+pBl5/1mSDs7lBwG/\nAnaQ1Cnp+z0dCEkHS7o/x3afpG1y+bvz614g6V5Jn+wW12mSLs91/0PSOyT9NJ+eeEDS+yu2f0zS\ntyXdJelFSWdJGp33Xyjpr5LWrNh+e0k35Oe+U1Jzt7/F8fk5F0r6i6R18urr8v2CHNcOkjbNx/x5\nSc9IOr+X49D1d58i6QlJ8yRNrVi/gqQjJT0i6VlJF0haq9u+B0n6J3BNL88xSdIdkl7I9UzM5QdW\n/A1mS/qvXL4acDmwfn49nfnv3Wsseb8DJD2e1303H/+P5XUj8t/piXz7qaQReV2zpA5JR0h6Evi1\npHsk7VlR94r5OL4fG1oiwjffGuoGPAZ8DLgYOCGXfQloy4/HAgEMr9inDfhSfvwFYBFwIKml4QTg\nn8DPgRHALsBCYGTe/py8/JG8/mfA9XndasCcXNdw4P3AM8C4in2fBz5EStBX7uH1XAecBqwMvA94\nGtipItbr+zgW+wBzgQ8AAjYltSqsCMwCjgZWAnbKr2GLirieAcbn570GeBQ4oOKYXNvtmN8EjAY2\nAJ4Cbsuvt2v/Y/K2GwDPAp/Ir/njeXndir/FI8DmwCp5+Yd9/O3OA77TdfyAHXs5Fl37npf/Lu/N\nx/Jjef1h+TU05b/jGcB53fY9N++7Sg/1b5v/lh/PsWwAvCuv2x3YJP8NPgq8BGyT1zUDHd3q6iuW\ncUAn6XTRSkAr8FrF6zgu7/t2YF3gBuD4iudaBJyU610FOBw4v+K5JwF3l/1/7Nvg30oPwDffqn3j\nzYRgy/wBvS7LnhA8XLHuvXn70RVlzwLvy4/PAaZXrBsJvA6MAfYF/t4tvjN488vxHODcPl7LmFzX\n6hVlPwDOqYi1r4TgSuCwHso/DDwJrFBRdh5wbEVcv6xY9zXg/m7HZEG3Y/7ZiuU/AKd32/9P+fER\nwG96iHNyxd/ifyrWfQW4oo+/3bnAmaR+FH29L7r2fVdF2Y+As/Lj+4GdK9atR/qiHV6x78Z91H8G\ncHLB9+ifuv4u9JwQ9BXL98jJQV63KvAqbyYEjwCfqFi/K+m0UtdzvUpF4gmsT0oG18jLFwGHD+b/\nrG+1cfMpA2tYEXEPcClw5HLs/q+Kxy/n+rqXjaxYnlPxvJ3AfNIH7YbAdrlpfIGkBcBngXf0tG8P\n1gfmR8TCirLHSb8+ixhD+oLoqd45EfFGH/V2f719vf5l2X5DYJ9ux2RH0pdelycrHr/Uw3NVOpz0\ny/vmfOrji31sC0se78dJx6Irrj9WxHQ/KRkb3cu+3fV2rJG0m6Sb8mmfBaTWkXV62rZALOuz5Pvt\nJVKC2mX9/Lp6eo0AT0fEvyv2fwL4B/CfSqfYdgN+10ds1qDcocQa3TGkpuufVJR1dcBbFXghP678\ngl4eY7oeSBoJrAU8Qfrg/ltEfLyPffu65OgTwFqSVq9ICt5JOg1QxBxSU3VP9Y6RtEJFUvBO4KGC\n9fbHHFILwcHLse9bjlVEPAl09avYEfirpOsiYlYvdYwBHsiP30k6Fl1xfTEi/tF9B0lje3v+Cj0e\n63z+/g+k0y2XRMRrkv5ESmJ6q7OvWOYBW1QsrwKsXbHJE6SE4t68XPkae3u+aaRWtOHAjRFR9P1l\nDcQtBNbQ8pfC+cChFWVPk75QPydpWP5F2dOX5rL4hKQdJa0EHA/cFBFzSC0Um0v6fO6staKkD0h6\nd8H455DOAf9A0sqStgIOAn5bMK5fkTpXjleyqVKHx5mkX96H55iagT2B6cvyopfTb4E9Je2aj//K\nubNbU4F9nwbeADbuKpC0T8W+z5G+8N7oYd8u35W0qqT3kPp2dHVC/AVwYj4+SFpX0qRleF1nAQdK\n2jl3CtxA0rtI5/lH5NgXSdqN1A+ly7+AtSW9raKsr1guIh2/D+b327G8mVxAOvXzP3mfdUinGJb2\nfvkTsA2p78K5y/CarYE4IbCh4DhSR7BKBwPfJjW1vof0pdsfvye1RswndcT7HED+Vb8LsB/pV9qT\nvNmhq6j9SeewnwD+SOp/8NciO0bEhcCJOb6FpA/+tSLiVVICsBup8+BpwAER8UBvdVVLTnImkTo0\nPk36NfxtCnwe5ebxE4F/5Ob07UkdJmdK6gRmkM7Nz+6jmr+ROlReDbRGxF9y+c/y/n+RtJDUMW+7\nZXhdN5MSjJNJfVf+BmyY3wOHAheQEpbP5Ofp2u8B0pf47Pya1u8rloi4l9QnYzowj9TB8CnglVzl\nCcCtwF3A3aQWsj4nzoqIl0mtGBuROuPaEKSIvlrAzMwaQ272fxRYMSIWlRtN9eRTVAuAzSLi0X7U\n8z1g84j4XNWCs7riFgIzszojac982mM10rDDu0kjPZa3vrVIp6LOrE6EVo+cEJiZ1Z9JpFNITwCb\nAfvFcjb3Kk10NQe4PCKuW9r21rgG5ZSBpLOBPYCnImLLivKvAYeQhtP8OSIOz+VHkbLV14FDI+LK\nXD6RdG5tGPCriPhhLt+IdD5tbaAd+Hw+R2pmZmYFDFYLwTnAxMoCSS2kLHfriHgPqdkLSeNIHbDe\nk/c5LfdEHkaaKW430kxd++dtIXXSOjkiNiV12jlowF+RmZlZAxmUeQgi4rqKcbxdvkyajvSVvM1T\nuXwSada3V4BHJc0iTQkKMKur97Ck6cAkSfeTpl39TN5mGmkYzulLi2udddaJsWO7h1WuF198kdVW\n694h3rrzcSrOx6oYH6fifKyKqcXj1N7e/kxErNvTujInJtoc+LCkE4F/A1Mj4hbSTGk3VWzXwZuz\np83pVr4d6TTBgopew5Xbv4WkKcAUgNGjR9Pa2lqFl1I9nZ2djBzZ16RsBj5Oy8LHqhgfp+J8rIqp\nxePU0tLyeG/rykwIhpNmc+saR3yBpI373qX/IuJMck/aCRMmRHNz80A/5TJpa2uj1mKqRT5OxflY\nFePjVJyPVTH1dpzKTAg6gItzz9ibJb1Bmtt7LhXTwJKu9tU1jWZP5c8CoyQNz60EldubmZlZAWUO\nO/wT+drvkjYnTe/5DGl2rv3yNb03Ig2puRm4BdhM0kZ5us79gBk5obgW2DvXOxm4ZFBfiZmZWZ0b\nlBYCSeeRLru5jqQO0hSvZwNnS7qHdDnOyfnL/V5JFwD3ka7bfUhEvJ7r+SrpMqnDgLPzFJ6QLqc6\nXdIJwO2kOcXNzMysoMEaZbB/L6t6nCIzIk4kzVfevfwy4LIeymfz5kgEMzMzW0aeqdDMzMycEJiZ\nmZkTAjMzM8MJgZmZmeGEwMzMzHBCYGZmZjghqAqperf29urVZWZmVpQTAjMzM3NCYGZmZk4IzMzM\nDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjMzMwMJwRmZmaGEwIz\nMzPDCYGZmZnhhMDMzMxwQmBmZmY4ITAzMzOcEJiZmRmDlBBIOlvSU5Lu6WHdtySFpHXysiSdImmW\npLskbVOx7WRJD+fb5Iry8ZLuzvucIkmD8brMzMwaxWC1EJwDTOxeKGkMsAvwz4ri3YDN8m0KcHre\ndi3gGGA7YFvgGElr5n1OBw6u2O8tz2VmZma9G5SEICKuA+b3sOpk4HAgKsomAedGchMwStJ6wK7A\nVRExPyKeA64CJuZ1a0TETRERwLnAXgP5eszMzBrN8LKeWNIkYG5E3NmthX8DYE7Fckcu66u8o4fy\n3p53CqnlgdGjR9PW1rb8LyJrbe13FYs1NXXS2tpWlbqq8NJqVmdnZ1X+dkOBj1UxPk7F+VgVU2/H\nqZSEQNKqwNGk0wWDKiLOBM4EmDBhQjQ3N/e7zpaWflexWGtrG1OnNlelroilb1Ov2traqMbfbijw\nsSrGx6k4H6ti6u04lTXKYBNgI+BOSY8BTcBtkt4BzAXGVGzblMv6Km/qodzMzMwKKiUhiIi7I+Lt\nETE2IsaSmvm3iYgngRnAAXm0wfbA8xExD7gS2EXSmrkz4S7AlXndC5K2z6MLDgAuKeN1mZmZ1avB\nGnZ4HnAjsIWkDkkH9bH5ZcBsYBbwS+ArABExHzgeuCXfjstl5G1+lfd5BLh8IF6HmZlZoxqUPgQR\nsf9S1o+teBzAIb1sdzZwdg/ltwJb9i9KMzOzocszFZqZmZkTAjMzM3NCYGZmZjghMDMzM5wQmJmZ\nGU4IzMzMDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjMzMwMJwRm\nZmaGEwIzMzPDCYGZmZnhhMDMzMxwQmBmZmY4ITAzMzOcEJiZmRlOCMzMzAwnBGZmZoYTAjMzM8MJ\ngZmZmTFICYGksyU9JemeirIfS3pA0l2S/ihpVMW6oyTNkvSgpF0ryifmslmSjqwo30jSzFx+vqSV\nBuN1mZmZNYrBaiE4B5jYrewqYMuI2Ap4CDgKQNI4YD/gPXmf0yQNkzQM+DmwGzAO2D9vC3AScHJE\nbAo8Bxw0sC/HzMyssQxKQhAR1wHzu5X9JSIW5cWbgKb8eBIwPSJeiYhHgVnAtvk2KyJmR8SrwHRg\nkiQBOwEX5f2nAXsN6AsyMzNrMMPLDiD7InB+frwBKUHo0pHLAOZ0K98OWBtYUJFcVG7/FpKmAFMA\nRo8eTVtbW39jp7W131Us1tTUSWtrW1XqqsJLq1mdnZ1V+dsNBT5Wxfg4FedjVUy9HafSEwJJ3wEW\nAb8bjOeLiDOBMwEmTJgQzc3N/a6zpaXfVSzW2trG1KnNVakroirV1KS2tjaq8bcbCnysivFxKs7H\nqph6O06lJgSSvgDsAewcsfjray4wpmKzplxGL+XPAqMkDc+tBJXbm5mZWQGlDTuUNBE4HPhkRLxU\nsWoGsJ+kEZI2AjYDbgZuATbLIwpWInU8nJETiWuBvfP+k4FLBut1mJmZNYLBGnZ4HnAjsIWkDkkH\nAacCqwNXSbpD0i8AIuJe4ALgPuAK4JCIeD3/+v8qcCVwP3BB3hbgCOCbkmaR+hScNRivy8zMrFEM\nyimDiNi/h+Jev7Qj4kTgxB7KLwMu66F8NmkUgpmZmS0Hz1RoZmZmTgjMzMzMCYGZmZnhhMDMzMxw\nQmBmZmY4ITAzMzOcEJiZmRlOCMzMzAwnBGZmZoYTAjMzM8MJgZmZmbGcCYGkFkkfrXYwZmZmVo5C\nCYGkv0n6UH58BDAd+L2kowcyODMzMxscRVsItgRuyo8PBlqA7YH/HoigzMzMbHAVvfzxCkBI2gRQ\nRNwHIGnNAYvMzMzMBk3RhOB64FRgPeCPADk5eGaA4jIzM7NBVPSUwReABcBdwLG57F3Az6ofkpmZ\nmQ22Qi0EEfEscHS3sj8PSERmZmY26IqOMhgh6URJsyU9n8t2kfTVgQ3PzMzMBkPRUwYnk0YafBaI\nXHYv8OWBCMrMzMwGV9FOhf8BbBoRL0p6AyAi5kraYOBCMzMzs8FStIXgVbolD5LWBZ6tekRmZmY2\n6IomBBcC0yRtBCBpPdIwxOkDFZiZmZkNnqIJwdHAo8DdwCjgYeAJ4LgBisvMzMwGUdFhh68C3wC+\nkU8VPBMRsZTdzMzMrE4UHXZ4gKStACLi6YgISVtL+nzB/c+W9JSkeyrK1pJ0laSH8/2auVySTpE0\nS9Jdkrap2Gdy3v5hSZMrysdLujvvc4okFT0AZmZmVvyUwfHAnG5lc4ATCu5/DjCxW9mRwNURsRlw\ndV4G2A3YLN+mAKdDSiCAY4DtgG2BYyqupXA66aJLXft1fy4zMzPrQ9GEYA3ghW5lz5P6EyxVRFwH\nzO9WPAmYlh9PA/aqKD83kpuAUbkT467AVRExPyKeA64CJuZ1a0TETfk0xrkVdZmZmVkBRechuA/4\nT+CCirL/AO7vx3OPjoh5+fGTwOj8eAOWbI3oyGV9lXf0UN4jSVNILQ+MHj2atra25X8FWWtrv6tY\nrKmpk9bWtqrUVYWXVrM6Ozur8rcbCnysivFxKs7Hqph6O05FE4IjgMsk7Qs8AmwK7Ax8ohpB5D4J\ng9JJMSLOBM4EmDBhQjQ3N/e7zpaWflexWGtrG1OnNlelrkbu9tnW1kY1/nZDgY9VMT5OxflYFVNv\nx6nQKYOIuB54L3ALsBpwM7BlRPyjH8/9r9zc3zWvwVO5fC4wpmK7plzWV3lTD+VmZmZWUNE+BETE\n4xHxw4g4JN9372S4rGYAXSMFJgOXVJQfkEcbbA88n08tXAnsImnN3JlwF+DKvO4FSdvn0QUHVNRl\nZmZmBRQ6ZZB7+E8F3geMrFwXER8psP95QDOwjqQO0miBHwIXSDoIeBz4dN78MtKpiFnAS8CB+Xnm\nSzqe1EoBcFxEdHVU/AppJMMqwOX5ZmZmZgUV7UPwe2AEqVPhS8v6JBGxfy+rdu5h2wAO6aWes4Gz\neyi/lXQ1RjMzM1sORROCDwLrRsQrAxmMmZmZlaNoH4K7WLLjnpmZmTWQoi0E1wBXSPo1ac6AxXIz\nvpmZmdWxognBh0kT/ny8W3nQwzl9MzMzqy9Fr3ZYxal3zMzMrNYUnodA0tqSPi/p23l5fUnuV2Bm\nZtYAil7++KPAg8Bnge/l4s3IVyI0MzOz+la0heCnwL4RMRFYlMtmki5DbGZmZnWuaEIwNiKuzo+7\nLpnzKsU7JZqZmVkNK5oQ3Cdp125lHwPurnI8ZmZmVoKiv/C/BVwq6c/AKpLOAPYEJg1YZGZmZjZo\nirYQ3AxsBdxLmnfgUWDbiLilz73MzMysLiy1hUDSMKATGBURPxr4kMzMzGywLbWFICJeBx4C1h74\ncMzMzKwMRfsQ/I7Uh+BnpCmMu0YaEBHXDERgZmZmNniKJgRfzvfHdisPYOOqRWNmZmalKNKHYAXg\nS8D1EfHKwIdkZmZmg61IH4I3gD85GTAzM2tcRYcdXidp+wGNxMzMzEpTtA/B48Dlki4B5rBkp8Lv\n9bqXmZmZ1YWiCcEqwJ/yY1/y2GyA6fuqWl2tm7fS8v2WftcTx8TSNzKzulUoIYiIAwc6EDMzMytP\noYRAUq9DCyNidvXCMTMzszIUPWUwi9RvoLIds6v9cFhVIzIzM7NBV/SUwRKjESS9AzgG+PtABGVm\nZmaDq+iwwyVExJPA14Ef9DcASd+QdK+keySdJ2llSRtJmilplqTzJa2Utx2Rl2fl9WMr6jkqlz8o\nadf+xmVmZjaULFdCkG0BrNqfJ5e0AXAoMCEitiSdftgPOAk4OSI2BZ4DDsq7HAQ8l8tPztshaVze\n7z3AROC0fJVGMzMzK6BQQiDp75Kuq7jdCswE/rcKMQwHVpE0nJRgzAN2Ai7K66cBe+XHk/Iyef3O\nkpTLp0fEKxHxKKnPw7ZViM3MzGxIUMTSxxZLmtyt6EXgzoh4uN8BSIcBJwIvA38BDgNuyq0ASBoD\nXB4RW0q6B5gYER153SPAdqSLLt0UEb/N5WflfS7q4fmmAFMARo8ePX769On9fQm0t/e7isWamjrp\n6BhZlbrGj69KNTWps7OTkSOrc5xqUfu86r2pmkY00fFKR7/rGb9eA7+haPz3VDX5WBVTi8eppaWl\nPSIm9LSuaKfCaUvfatlJWpP0634jYAFwIanJf8BExJnAmQATJkyI5ubmftfZ0v85XxZrbW1j6tTm\nqtRVINerW21tbVTjb1erqjGRUJfWzVuZ+tDUftcT+zfwG4rGf09Vk49VMfV2nIqeMrhY0oe7lX1Y\n0lt+gS+jjwGPRsTTEfEacDHwIWBUPoUAaWbEufnxXGBMfv7hwNuAZyvLe9jHzGzIk1S1W3t7e1Xq\nsdpStFPhR4EbupXdCPT3Z8w/ge0lrZr7AuwM3AdcC+ydt5kMXJIfz8jL5PXXRDrnMQPYL49C2AjY\nDLi5n7GZmZkNGUUnJvo3sBrwQkXZSOC1/jx5RMzMrQy3AYuA20nN+X8Gpks6IZedlXc5C/iNpFnA\nfNLIAiLiXkkXkJKJRcAhEfF6f2IzMzMbSoomBFcCZ0j6r4h4QdIawKnAFf0NICKOIU1yVGk2PYwS\niIh/A/v0Us+JpM6JZmZmtoyKnjL4FrAGMF/SU6Rf528jTU5kZmZmda7oKIPngN3zlMVjgDl5tkIz\nMzNrAEWvdrgL8FhEPAQ8mcu2AN4ZEVcNYHxmZmY2CIqeMvg5sLBb2cJcbmZmZnWuaELw9oiY161s\nHvCOKsdjZmZmJSiaEMyWtFO3smbg0eqGY2ZmZmUoOuzwWODifI2AR4BNgAPzzczMzOpcoRaCiLgE\n2IU0OdHu+X7XXG5mZmZ1rmgLARFxM54O2MzMrCEttYVA0lhJ50iaK+mVfD9N0saDEaCZmZkNvD4T\nAknvJl1n4O3Ad4BP5vt1gVvzejMzM6tzSztl8EPg5xHx3W7l5+QLD/0I2HNAIjMzM7NBs7SE4CO8\nebnh7n6Chx2amZk1hKX1IRhG75c4fi2vNzMzszq3tITgFnqfa+ALwK1VjcbMzMxKsbRTBt8FrswX\nMrqINF3xesA+pFMJuw5seGZmZjYY+mwhiIgbSBMSbQ1cDTyQ77cGJub1ZmZmVueWOjFRRNwIfETS\nKsBawHMR8dKAR2ZmZmaDZllmKnwZmDuAsZiZmVlJil7t0MzMzBqYEwIzMzPrPSGQ9OOKxzsNTjhm\nZmZWhr5aCKZUPP7TQAdiZmZm5emrU+Gdki4C7gNGSDqup40i4nsDEpmZmZkNmr5aCPYG7iBNRCRg\nTA+3pv4GIGmUpIskPSDpfkk7SFpL0lWSHs73a+ZtJekUSbMk3SVpm4p6JuftH5bU2/UXzMzMrAe9\nthBExFPACQCShkdEb1MY99fPgCsiYm9JKwGrAkcDV0fEDyUdCRwJHAHsBmyWb9sBpwPbSVoLOAaY\nAATQLmlGRDw3QDGbmZk1lEKjDCLiQElrSjpA0lH5fq3+Prmkt5GuqHhWfp5XI2IBMAmYljebBuyV\nH08Czo3kJmCUpPVIUyhfFRHzcxJwFTCxv/GZmZkNFYqIpW8k7QD8mTR18ePAO4F3A7vnmQyX78ml\n9wFnkvopbA20A4cBcyNiVN5GpNkRR0m6FPhhRFyf111NajloBlaOiK4Wje8CL0dEaw/POYXcYXL0\n6NHjp0+fvrzhL9be3u8qFmtq6qSjY2RV6ho/virV1KTOzk5GjqzOcapF7fOq96ZqGtFExysd/a5n\n/HoN/IZiCLynqvhB1dTUREdHFd5TjfwhRW2+p1paWtojYkJP64rOVPhT4CsRsfjbU9K+wCnAB/oR\n23BgG+BrETFT0s9IpwcWi4iQtPSspaCIOJOUhDBhwoRobm7ud50tLf2uYrHW1jamTm2uSl0Fcr26\n1dbWRjX+drWq5fvVe1O1bt7K1Iem9rue2L+B31AMgfdUFT+oWltbmTq1Cu+pRv6Qov7eU0UnJtoc\nuKBb2UXApv18/g6gIyJmVtS5DfCvfCqAfP9UXj+X1JmxS1Mu663czMzMCiiaEDwM7NetbB/gkf48\neUQ8CczJl1cG2Jl0+mAG6fLK5PtL8uMZwAF5tMH2wPMRMQ+4Etgl93NYk3SFxiv7E5uZmdlQUvSU\nwdeBSyUdSupDMJbU03+PKsTwNeB3eYTBbOBAUqJygaSD8vN9Om97GfAJYBbwUt6WiJgv6Xjglrzd\ncRExvwqxmZmZDQmFEoKIuEHSJsDuwPrA/wGXVeNLNyLuIA0X7G7nHrYN4JBe6jkbOLu/8ZiZmQ1F\ny3L54+eA3w5gLGZmZlYSX+3QzMzMnBCYmZmZEwIzMzNjGRICSRsOZCBmZmZWnmVpIbgdIA89NDMz\nswbS5ygDSe2k6wvcDgzLxceSpiw2MzOzBrG0FoK9gb8AGwKrSroNGCGpJV+p0MzMzBrA0hKCYRFx\nUUQcCSwkXX5YpNkF75D08EAHaGZmZgNvaRMT/U7SO0nXF1gZWBP4d0R8CkDSWgMcn5mZmQ2CPhOC\niNhO0nDgvcD1wKnA6pJOB27LN18zwMzMrM4tdZRBRCyKiNuBVyPiI8CLQBvp4kYnDWx4ZmZmNhgK\nX8sA+Ea+j4g4Hzh/AOIxMzOzEhSehyAizskPNx6YUMzMzKwsyzx1cb7qoZmZmTUQX8vAzMzMnBCY\nmZmZEwIzMzPDCYGZmZnhhMDMzMxwQmBmZmY4ITAzMzOcEJiZmRlOCMzMzAwnBGZmZkaNJASShkm6\nXdKleXkjSTMlzZJ0vqSVcvmIvDwrrx9bUcdRufxBSbuW80rMzMzqU00kBMBhwP0VyycBJ0fEpsBz\nwEG5/CDguVx+ct4OSeOA/YD3ABOB0yQNG6TYzczM6l7pCYGkJmB34Fd5WcBOwEV5k2nAXvnxpLxM\nXr9z3n4SMD0iXomIR4FZwLaD8wrMzMzqnyKi3ACki4AfAKsDU4EvADflVgAkjQEuj4gtJd0DTIyI\njrzuEWA74Ni8z29z+Vl5n4u6PR2SpgBTAEaPHj1++vTp/X4N7e39rmKxpqZOOjpGVqWu8eOrUk1N\n6uzsZOTI6hynWtQ+r3pvqqYRTXS80tHvesav18BvKIbAe6qKH1RNTU10dFThPdXIH1LU5nuqpaWl\nPSIm9LRu+GAHU0nSHsBTEaErL3gAABbqSURBVNEuqXkwnjMizgTOBJgwYUI0N/f/aVta+l3FYq2t\nbUyd2lyVukrO9QZUW1sb1fjb1aqW71fvTdW6eStTH5ra73pi/wZ+QzEE3lNV/KBqbW1l6tQqvKca\n+UOK+ntPlZoQAB8CPinpE8DKwBrAz4BRkoZHxCKgCZibt58LjAE6JA0H3gY8W1HepXIfMzMzW4pS\n+xBExFER0RQRY0mdAq+JiM8C1wJ7580mA5fkxzPyMnn9NZFSzBnAfnkUwkbAZsDNg/QyzMzM6l7Z\nLQS9OQKYLukE4HbgrFx+FvAbSbOA+aQkgoi4V9IFwH3AIuCQiHh98MM2MzOrTzWTEEREG9CWH8+m\nh1ECEfFvYJ9e9j8ROHHgIjQzM2tcpQ87NDMzs/I5ITAzMzMnBGZmZuaEwMzMzHBCYGZmZjghMDMz\nM5wQmJmZGU4IzMzMDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjM\nzMwMJwRmZmaGEwIzMzPDCYGZmZnhhMDMzMxwQmBmZmY4ITAzMzOcEJiZmRlOCMzMzAwnBGZmZkbJ\nCYGkMZKulXSfpHslHZbL15J0laSH8/2auVySTpE0S9JdkrapqGty3v5hSZPLek1mZmb1qOwWgkXA\ntyJiHLA9cIikccCRwNURsRlwdV4G2A3YLN+mAKdDSiCAY4DtgG2BY7qSCDMzM1u6UhOCiJgXEbfl\nxwuB+4ENgEnAtLzZNGCv/HgScG4kNwGjJK0H7ApcFRHzI+I54Cpg4iC+FDMzs7qmiCg7BgAkjQWu\nA7YE/hkRo3K5gOciYpSkS4EfRsT1ed3VwBFAM7ByRJyQy78LvBwRrT08zxRS6wKjR48eP3369H7H\n3t7e7yoWa2rqpKNjZFXqGj++KtXUpM7OTkaOrM5xqkXt86r3pmoa0UTHKx39rmf8eg38hmIIvKeq\n+EHV1NRER0cV3lON/CFFbb6nWlpa2iNiQk/rhg92MD2RNBL4A/D1iHgh5QBJRISkqmUtEXEmcCbA\nhAkTorm5ud91trT0u4rFWlvbmDq1uSp11UiuNyDa2tqoxt+uVrV8v3pvqtbNW5n60NR+1xP7N/Ab\niiHwnqriB1VraytTp1bhPdXIH1LU33uq7D4ESFqRlAz8LiIuzsX/yqcCyPdP5fK5wJiK3ZtyWW/l\nZmZmVkDZowwEnAXcHxH/W7FqBtA1UmAycElF+QF5tMH2wPMRMQ+4EthF0pq5M+EuuczMzMwKKPuU\nwYeAzwN3S7ojlx0N/BC4QNJBwOPAp/O6y4BPALOAl4ADASJivqTjgVvydsdFxPzBeQlmZmb1r9SE\nIHcOVC+rd+5h+wAO6aWus4GzqxedmZnZ0FF6HwIzMzMrnxMCG1xSdW7t7dWry8zMnBCYmZmZEwIz\nMzOj/FEGZmZmNaNNbVWrq7O1k7aW/tfXHM39rqMItxCYmZmZEwIzMzNzQmBmZmY4ITAzMzOcEJiZ\nmRlOCMzMzAwnBGZmZoYTAjMzM8MJgZmZmeGEwMzMzHBCYGZmZjghMDMzM5wQmJmZGU4IzMzMDCcE\nZmZmBgwvOwAzs/5oa1NV6unsbKWtraUqdTU3R1XqMRtMbiEwMzMzJwRmZmbmhMDMzMxwQmBmZmY0\nWEIgaaKkByXNknRk2fGYmZnVi4ZJCCQNA34O7AaMA/aXNK7cqMzMzOpDwyQEwLbArIiYHRGvAtOB\nSSXHZGZmVhcU0RjjZSXtDUyMiC/l5c8D20XEV7ttNwWYkhe3AB4c1ECXbh3gmbKDqAM+TsX5WBXj\n41Scj1UxtXicNoyIdXtaMeQmJoqIM4Ezy46jN5JujYgJZcdR63ycivOxKsbHqTgfq2Lq7Tg10imD\nucCYiuWmXGZmZmZL0UgJwS3AZpI2krQSsB8wo+SYzMzM6kLDnDKIiEWSvgpcCQwDzo6Ie0sOa3nU\n7OmMGuPjVJyPVTE+TsX5WBVTV8epYToVmpmZ2fJrpFMGZmZmtpycEJiZmZkTAjMzM3NCYGZDlKQR\nRcrMhgonBCXLwyRXrlheRdLY8iKqXZIOkTSqYnlNSV8pM6ZaJmnVsmOocTcWLDNA0j5Fymzx5/gW\nZcexrJwQlO9C4I2K5ddzmb3VwRGxoGshIp4DDi4xnpok6YOS7gMeyMtbSzqt5LBqhqR3SBoPrCLp\n/ZK2ybdmwElU744qWDakSdoTuAO4Ii+/T1JdzInTMPMQ1LHh+WJMAETEq3liJXurYZIUeaxsvsKl\nj9VbnQzsSp6YKyLulPSRckOqKbsCXyDNZvq/FeULgaPLCKiWSdoN+ASwgaRTKlatASwqJ6qadizp\nYnttABFxh6SNygyoKCcE5Xta0icjYgaApEnU3sUwasUVwPmSzsjL/5XLrJuImCOpsuj1smKpNREx\nDZgm6T8j4g9lx1MHngDagU/m+y4LgW+UElFtey0inu/2/1cXE/44ISjffwO/k3QqIGAOcEC5IdWs\nI0hJwJfz8lXAr8oLp2bNkfRBICStCBwG3F9yTDUnIv4gaXfgPcDKFeXHlRdV7YmIO4E7Jf02Itwi\nsHT3SvoMqUVzM+BQ4IaSYyrEMxXWCEkjASKis+xYrL5JWgf4GfAxUpL5F+CwiHi21MBqjKRfkPoM\ntJASy72BmyPioFIDqzGS7qaPX7gRsdUghlPzcmfe7wC75KIrgeMj4pXyoirGCUFJJH0uIn4r6Zs9\nrY+I/+2pfCiSdEFEfLq3DyZ/INnykHRXRGxVcT8SuDwiPlx2bLVE0oZ9rY+IxwcrlnogaZ+IuHBp\nZbXIpwzKs1q+X73UKOrDYfl+j1KjqBOS1iWNvhhLxf94RHyxrJhq1Mv5/iVJ6wPPAuuVGE9N8hf+\nMjuKt44U66ms5jghKElEnJHvv999nUcZLCki5uWHq0XEfZXr8lAxf2At6RLg78BfcWfCvlya57X4\nMXAbqfXJfVJ6IWkhb7bQrQSsCLwYEWuUF1XtaITRGD5lUDJJbcAXIuKxvPwB4FcRsXWZcdUiSfcA\nvwF+ROoE9iNgQkTsUGpgNUbSHRHxvrLjqCd5hsKVI+L5smOpB0pd6CcB20fEkWXHUwskbQ28DzgO\n+F7FqoXAtXnelJrmhKBkknYldQA7BdgA2A34UkTcVmpgNUjSasBJwHjSqZbfASdFxBt97jjESDoB\nuCEiLis7llqWO399C3hnRByce4RvERGXlhxa3ZB0e0S8v+w4aomkFSPitbLjWB4+ZVCyiLhS0n+T\nhtA9A7w/Ip4sOaxa9RrpvO8qpBaCR50M9Ogw4GhJrwKvkkYahJt23+LXpHH1XS1Mc0nneZ0Q9EDS\npyoWVwAmAP8uKZxaNlbSD4BxLDmcdePyQirGCUHJJH0X+DTwEWAroE3StyLiz+VGVpNuIZ0f/wCw\nDvCLPLmM51OvEBHuqFrMJhGxr6T9ASLiJXWbTcaWsGfF40XAY6TTBrakXwPHkGYMbQEOpE4uE+CE\noHxrA9tGxMvAjZKuIHVsckLwVgdFxK358TxgkqTPlxlQLcpfap8FNoqI4yWNAdaLiJtLDq3WvCpp\nFXJHOUmbADU/VrwsEXFg2THUiVUi4uo8zfrjwLGS2lmyX0FNqouspZFFxNeBNSTtIWkP4OWI+HjZ\ncdWoOyUdKumifPsqML3soGrQaaRm8M/k5U7g5+WFU7OOIU19PUbS74CrgcPLDal2SfqRpDUkrSjp\naklPS/pc2XHVoFckrQA8LOmrkv4DGFl2UEU4IShZvnzozcA+pFMHMyXtXW5UNet0UofC0/Kt67Et\nabuIOIR8fjf3bvZQ1gr5A3tN4FOkCx2dRxqx0lZiWLVul4h4gTQfyGPApsC3S42oNh1GmgHzUNJn\n1OeByaVGVJBPGZTvf4APRMRTsHhSmb8CF5UaVW36QLfhmNdIurO0aGrXa/lKkF1N4euy5CW2h7yI\neEPS4RFxAT49V1TX98XuwIU9XMDHgIi4JT/sJPUfqBtOCMq3QlcykD2LW25687qkTSLiEQBJG+OJ\nd3pyCvBH4O2STiTN0f8/5YZUk/4qaSpwPvBiV2FEzC8vpJp2qaQHSCN9vpwTTY8y6EbS5qSWkw1Z\ncqbQnUoLqiDPQ1AyST8CtiY1WQLsC9wVEUeUF1VtkrQzqQfvbNJQug2BL0bENaUGVoMkvQvYmXSc\nro4IX+2wG0mP9lAc9TA8rCyS1gKej4jX8zwOa3iY9JJyq+UvSENaF/9giYj2XneqEU4ISibpJGAm\nsGMu+jtp9i8nBN3k2eQAtsj3DwLUw1XEBpOk7YF7I2JhXl4DeHdEzCw3Mqt3+bLaY1nyl++5pQVU\ngyS1R8T4suNYHk4ISibptojYplvZXb6C31v1cqzeUjbUSbod2CbyP3fuQHerj9Nb+QuuOEm/ATYB\n7uDNX74REYeWF1XtyK0nkDoTPkU6bbf4x0o9nIpyH4KSSPoy8BVgY0l3VaxaHfhHOVHVJknvIE3r\nvIqk95OawSFdNGTV0gKrXYqKTD93oPP/eje9fcEBTgh6NgEYF/4V2Zt20vun6/OpcgRGADV/Ksof\nEuX5PXA58AOg8uIgC+shkxxku5KGhjUBP+HNf7gXgKNLiqmWzZZ0KGmYJqTEc3aJ8dQqf8Etm3uA\nd5AmBbNuImKjIttJ+nhEXDXQ8SwPnzKwupGnKf5DH+snR8S0wYypFkl6O2mkwU6kXyZXA1/vNppl\nyJN0IXBoxeW1rQ+SriVdze9mlmwK/2RpQdWhWj7N6YTAGkYt/6NZ7ZD0f6REaXX8BVeYpI/2VB4R\nfxvsWOpZLV8h0qcMrJF4lhQWD2U9gTRe/ArSRbO+ERG/LTWw2tFKeq+cBOxVUd5VZj3wF3/V1Oyv\ncCcE1khq9h9tkO0SEYfnOdQfI03Pex3ghIA3v9jydeuX+JLLFzuyCpKuj4gdJS1kyf8xX1a7wTgh\nsEbiFoLEU8z2wSN8lk1E7JjvfVnt6nis7AB644TAGok/zBNPMds3j/CxAZMvdXw28Pt8YbElRMSn\nBj+qYtyp0OqGpNHA/wPWj4jdJI0DdoiIs0oOreZ4ilmzckjalHRRo32BW0nTrf+lHoa3OiGwuiHp\nctI/13ciYus82c7tEfHekkOrKZJWJjWJ70g653s9cHpEuJXAbJDkGUL3IM0H8jrps+tntdwK5avq\nWT1ZJ1+u9g2AiFiEr3bYk3OB9wD/H3AqMA74TakRmQ0hkrYiTaL2Y+APwD6kidRq+kJs7kNg9eRF\nSWuTezrni/g8X25INWnLiBhXsXytpPtKi8ZsCMl9CBYAZwFHVlx8baakD5UX2dI5IbB68k1gBrCJ\npH8A6wJ7lxtSTbpN0vYRcROApO1I5zLNbODtExFLTBUuaaOIeLSWOxSCEwKrE5KGAR/Nty1IQwwf\njIjXSg2shki6m9R6siJwg6R/5uUNgQfKjM1sCLkI6D5j6kVAzV8S2QmB1YXcW37/iDgZuLfseGrU\nHhWP1wQ+nB9fR2rCNLMBIuldpL47b5NU2RKwBrByOVEtGycEVk/+IelU4Hzgxa7CiLitvJBqR0Q8\nDiDpMOBLwMWklpTfAL8kdTI0s4GxBSkpHwXsWVG+EDi4lIiWkYcdWt3IV1vrLiJip0EPpobl2fd2\niIgX8/JqwI0RsVW5kZk1Pkk7RMSNZcexPNxCYHUjIlrKjqFOiCWHY76Op3U2G1CSDo+IHwGfkbR/\n9/URcWgJYS0TJwRWNzxTYWG/Jg1x+mNe3os0BMrMBs79+b5uR/T4lIHVDc9UWJykbUgzFQL8PSJu\nLzMes6FC0j4RceHSymqREwKrG5JuiYgPSLo9It6fy+6IiPeVHZuZGYCk2yJim6WV1SKfMrB64pkK\nzawmSdoN+ASwgaRTKlatASwqJ6pl44TA6sm38EyFZlabniD1H/gk0F5RvhD4RikRLSOfMrC6kvsN\neKZCM6tJklas188lX+3Q6kYeX3848O+IuKde/+nMrKFtK+kqSQ9Jmi3pUUmzl75b+dxCYHVD0obA\nvvn2BmnGwgsi4p+lBmZmlkl6gHSKoJ2K+UAi4tnSgirICYHVJUmbAd8FPhsRw8qOx8wMQNLMiNiu\n7DiWhzsVWl3p1krwOukUgplZrbhW0o9J1xJ5pauwHq654hYCqxuSZpIu7XshcH73a46bmZWtnq+5\n4oTA6oakLSLiwbLjMDNrRB5lYPVkgaSz8hTGSBon6aCygzIz6yJpdL1+TjkhsHpyDnAlsH5efgj4\nemnRmJm91TnU6eeUEwKrJ+tExAWkIYdExCKWvMyvmVnZ6vZzygmB1RNfy8DMal3dfk552KHVk2/i\naxmYWW2r288pJwRWTzYBdgPGAP8JbIffw2ZWQyLiNkkfpQ6vueJhh1Y3JN0VEVtJ2hE4HmgFvlev\ns4KZWeOQ9Km+1kfExYMVy/LyryurJ10dc3YHfhkRf5Z0QpkBmZlle+b7twMfBK7Jyy3ADaSZC2ua\nEwKrJ3MlnQF8HDhJ0gjcMdbMakBEHAgg6S/AuIiYl5fXIw1FrHn+MLV68mnS+N5dI2IBsBbw7XJD\nMjNbwpiuZCD7F/DOsoJZFu5DYGZmViWSTgU2A87LRfsCsyLia+VFVYwTAjMzsyrKHQw/nBevi4g/\nlhlPUU4IzMzMzJ0KzczM+kvS9RGxo6SF5FkKu1aRLn+8RkmhFeYWAjMzM/MoAzMzM3NCYGZmZjgh\nMDMzM5wQmFk3kj4j6VZJnZLmSbo8Xz9iafuFpE0HI0Yzqz4nBGa2mKRvAj8F/h8wmjTD2mnApDLj\n6oskj5YyqwInBGYGgKS3AccBh0TExRHxYkS8FhH/FxHflrStpBslLcgtB6dKWinve12u5s7csrBv\nLt9D0h15nxskbVXxfNtIul3SQkkXSjq/8mJVkg6WNEvSfEkzJK1fsS4kHSLpYeBhST+X9JNur2eG\npG8M3BEzayxOCMysyw7AykBvs6q9DnwDWCdvuzPwFYCI+EjeZuuIGBkR50t6P3A28F/A2sAZwAxJ\nI3Ii8UfSRV/WIk3z+h9dTyRpJ+AHpOtXrAc8DkzvFs9ewHbAOGAasL+kFfL+6wAfA36/PAfCbChy\nQmBmXdYGnomIRT2tjIj2iLgpIhZFxGOkL/iP9lHfFOCMiJgZEa9HxDTgFWD7fBsOnJJbIS4Gbq7Y\n97PA2RFxW0S8AhwF7CBpbMU2P4iI+RHxckTcDDxPSlIA9gPaIuJfy3QEzIYwJwRm1uVZYJ3ezslL\n2lzSpZKelPQCqZ/BOn3UtyHwrXy6YIGkBcAYYP18mxtLzow2p+Lx+qRWAQAiojPHt0Ev20NqJfhc\nfvw54Dd9xGZm3TghMLMuN5J+we/Vy/rTgQeAzfI0rEeTpmXtzRzgxIgYVXFbNSLOA+YBG0iq3H9M\nxeMnSAkFAJJWI7VgzK3Ypvs0q78FJknaGng38Kc+YjOzbpwQmBkAEfE88D3g55L2krSqpBUl7Sbp\nR8DqwAtAp6R3AV/uVsW/gI0rln8J/Lek7ZSsJml3SauTko/Xga9KGi5pErBtxb7nAQdKep+kEaTW\niJn5VEVv8XcAt5BaBv4QES8v/9EwG3qcEJjZYhHxE+CbwP8AT5N+5X+V9Gt7KvAZYCHpy/78brsf\nC0zLpwc+HRG3AgcDpwLPAbOAL+TneRX4FHAQsIDUxH8pqYWCiPgr8F3gD6TWhE1I/QKWZhrwXny6\nwGyZ+eJGZlYTJM0EfhERv+5HHR8hnTrYMPzhZrZM3EJgZqWQ9FFJ78inDCYDWwFX9KO+FYHDgF85\nGTBbdp7hy8zKsgVwAbAaMBvYOyLmLU9Fkt4N3ArcCRxYtQjNhhCfMjAzMzOfMjAzMzMnBGZmZoYT\nAjMzM8MJgZmZmeGEwMzMzID/H6MdSJY2+/XoAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gxgnEGCJsFz2",
        "outputId": "7b94b177-e220-480b-8a25-36a070fa83ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 370
        }
      },
      "source": [
        "# How many comments have multi-labels?\n",
        "rowsums = df.iloc[:,2:].sum(axis=1)\n",
        "x=rowsums.value_counts()\n",
        "#plot\n",
        "plt.figure(figsize=(8,5))\n",
        "ax = sns.barplot(x.index, x.values)\n",
        "plt.title(\"Multiple categories per comment\")\n",
        "plt.ylabel('# of Occurrences', fontsize=12)\n",
        "plt.xlabel('# of Categories', fontsize=12)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, '# of Categories')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAFQCAYAAADePFZZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5gdVZ3v//dHEESUe0QgQFBQBxkd\nNQLqzysKQcV4HFQ4KpFBGcfb6M9RUWfEQThHx5lBOSozDCCgjIh4ARVFjop3kAAqAioRxCSCBBJu\noiDyPX/Uim7armRL0r073e/X8+xnV61aVetb3YH69lqrqlJVSJIkjec+ow5AkiRNXSYKkiSpl4mC\nJEnqZaIgSZJ6mShIkqReJgqSJKmXiYK0liSpJDuvYvtlSZ425LF+nuSZay24KS7Jk5P8ZNRxSPpT\nJgqa8dpF+c4kW40pv6Rd/Ofci2OelOTIwbKqemRVnbdGwU6QJE9LsmRU7VfVN6vq4aNqX5BkTvv3\nvv6oY9HUYqIgda4GDly5kuQvgfuPLpyZY124MK0LMUoTxURB6nwUOGhgfQFwymCFJOclecXA+suT\nfGvsgZIcCrwEeEuS25J8rpX/YTghybuSnJHkE0luTXJxkkePF1iS+yQ5LMnPktyY5PQkW/SdSJL5\nSb6f5Ja2z7xWfnCSK1p7VyX521a+MfBFYNsW721Jtl1du0kOSnJN2/ZPY85vwyTvT/LL9nl/kg3b\ntqclWZLkrUmuAz4ytkejtf+pJMuSXJ3k9QPbdk+ysJ3fr5L8e8/PYWU7b09yQ4vvJQPbN0zyr0l+\n0Y7zH0k26ouxp41XDvxML0/y2Fb+F+3fy01tyOl5A/uclOTDSb7YftbfTvLg9jNakeTHSR4zUP/n\nSd6c5IdJfp3khCRbt/1vTfJ/k2w+UH/PJN9pbf8gA8NdLaZ3tzZvTfLl/LEn7Rvt+6YW1xPGO2fN\nPCYKUud8YJP2P/j1gAOAj92bA1XVccCpwL9U1QOqar+eqvOBTwJbAP8NfDbJfcep9zrg+cBTgW2B\nFcCHxjtgkt3pEpw3A5sBTwF+3jZfDzwX2AQ4GDg6yWOr6tfAvsAvW7wPqKpfrqrdJLsCH6ZLiLYB\nNgW2GwjlHcCewF8BjwZ2B/5xYPuD23nvCBw65hzuA3wO+EE75l7AG5Ls06p8APhAVW0CPBQ4fbyf\nxUA7W7XjLACOS7JyiOM9wMNajDu3Ou8cJsYW5wuBd9ElmJsAzwNubL/DzwFfBh5E93M8daBdgBe1\nn8dWwB3Ad4GL2/oZwNjk56+BZ7V496NL7N4OzKL7//jrW0zbAV8Ajmyx/wPwqSSzBo71P+l+/w8C\nNmh1oPu3ArBZ+zfw3bHnrJnJREH6o5W9Cs8CrgCWTnB7F1XVGVX1O7oLw/3oLq5jvQp4R1Utqao7\n6C5O+2f87vBDgBOr6tyquruqllbVjwGq6gtV9bPqfJ3uQvbkVcS3qnb3Bz5XVd+qqjvpLrCDL455\nCXBEVV1fVcuAfwZeNrD9buDwqrqjqn4zpt3HA7Oq6oiqurOqrgL+iy55A/gdsHOSrarqtqo6fxXn\nAPBPrZ2v011EX5QkdBf/N1bV8qq6FfhfA22sLkaAV9Algxe2n+miqrqG7nf4AOA9Lf6vAp9nYGgL\n+ExVXVRVvwU+A/y2qk6pqt8DnwAeM6at/1NVv6qqpcA3gQuq6pKB/VfWfylwdlWd3X7/5wILgWcP\nHOsjVfXTdk6n0yVKUi/H3aQ/+ihd9+tOjBl2mCCLVy5U1d2t633bcertCHwmyd0DZb8HtuZPk5nt\ngbPHayzJvsDhdH+V3oduDsalq4hvVe1uOyb+25PcOFBvW+CagfVruOe5LWsXub52t01y00DZenQX\nSOiSoSOAHye5Gvjnqvp8z7FWtB6TsXHMojv/i7qcAYC0doaJEbqf9c/GKd8WWFxVgz+3a7hnj8uv\nBpZ/M876A8Ycc9j6OwIvTDLYi3Vf4GsD69cNLN8+TlvSPZgoSE1VXdMuPM+muxiN9WvuOcHxwas6\n3BBNbr9yoXW3zwZ+OU69xcDfVNW3hzjmYrru+Hto8wM+RddjcmZV/S7JZ+kujn3x9rab5Frg4QPr\nGwFbDlT5Jd1F67K2vgP3PLdV/XwWA1dX1S7jbayqK4ED28/sBcAZSbYckxCstHmSjQe27QD8CLiB\n7gL7yPZX+rhNrSLGlXH+yc+a7jy3T3KfgWRhB+Cnqzne2rAY+GhVvfJe7OurhDUuhx6kezoEeEbP\nRef7wAuS3D/d8xLGSyZW+hXwkNW09bgkL2hd+W+gG6serxv9P4CjkuwIkGRWkvk9xzwBODjJXukm\nI26X5BF0Y9EbAsuAu1rvwt5j4t0yyaZDtnsGsF+SJybZgG5YIgP7fhz4x7bPVnRDE8PO+fgecGub\nSLhRkvWS7Jbk8S2OlyaZ1S7CK3sd7u49Gvxzkg2SPJlujsYn277/RTdP40HtuNsNzIMYxvHAPyR5\nXDo7t5/VBXR/qb8lyX3bZML9gNP+jGPfWx+j+73s035u90s3MXP2EPsuo/s5ru7frWYYEwVpQBvD\nX9iz+WjgTrqL6sl0Exb7nADs2maef7anzpnAi+kmCb4MeEGbrzDWB4CzgC8nuZUumdijJ/7v0SYq\nAjcDXwd2bGPwr6cbk15BN6HtrIH9fkx3cb+qxbztqtqtqsvoJumdBlwL3EY3WfKOdsgj6cbGf0g3\nvHFxK1utNk7/XLqx86vp/vo/nm7CJMA84LIkt7UYD+iZQwBdN/sKur/yTwVetXLOBvBWYBFwfpJb\ngP/LQC/JEHF+EjiKbiLqrcBngS3anI396CaI3kA36fOggXYnTFUtppsk+3a6C/9iuomtq/1/fVXd\nTnc+327/BsabL6MZKFX2NkmTLcm7gJ2r6qWjjmVtSPIAur/ud6mqq0cdD3S3OAIfq6ph/pqW1MMe\nBUn3SpL92jDMxsC/0vUc/Hy0UUla20wUJN1b8+m69H8J7EI3BGAXpTTNOPQgSZJ62aMgSZJ6mShI\nkqRePnBpHFtttVXNmTNn1GFIkjQpLrroohuqatZ420wUxjFnzhwWLuy7lV6SpOklyTV92xx6kCRJ\nvUwUJElSLxMFSZLUy0RBkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS\n1Mt3PfwZHvfmU0Ydwlpz0fsOGnUIkqR1gD0KkiSp16QkCklOTHJ9kh+Ns+1NSSrJVm09SY5JsijJ\nD5M8dqDugiRXts+CgfLHJbm07XNMkrTyLZKc2+qfm2TzyThfSZKmi8nqUTgJmDe2MMn2wN7ALwaK\n9wV2aZ9DgWNb3S2Aw4E9gN2Bwwcu/McCrxzYb2VbhwFfqapdgK+0dUmSNKRJSRSq6hvA8nE2HQ28\nBaiBsvnAKdU5H9gsyTbAPsC5VbW8qlYA5wLz2rZNqur8qirgFOD5A8c6uS2fPFAuSZKGMLI5Cknm\nA0ur6gdjNm0HLB5YX9LKVlW+ZJxygK2r6tq2fB2w9dqJXpKkmWEkdz0kuT/wdrphh0lRVZWk+rYn\nOZRuqIMddthhssKSJGlKG1WPwkOBnYAfJPk5MBu4OMmDgaXA9gN1Z7eyVZXPHqcc4FdtaIL2fX1f\nQFV1XFXNraq5s2bNWoNTkyRp+hhJolBVl1bVg6pqTlXNoRsueGxVXQecBRzU7n7YE7i5DR+cA+yd\nZPM2iXFv4Jy27ZYke7a7HQ4CzmxNnQWsvDtiwUC5JEkawmTdHvlx4LvAw5MsSXLIKqqfDVwFLAL+\nC3g1QFUtB94NXNg+R7QyWp3j2z4/A77Yyt8DPCvJlcAz27okSRrSpMxRqKoDV7N9zsByAa/pqXci\ncOI45QuB3cYpvxHY688MV5IkNT6ZUZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1MlGQJEm9TBQk\nSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RB\nkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkCRJvUwU\nJElSr0lJFJKcmOT6JD8aKHtfkh8n+WGSzyTZbGDb25IsSvKTJPsMlM9rZYuSHDZQvlOSC1r5J5Js\n0Mo3bOuL2vY5k3G+kiRNF5PVo3ASMG9M2bnAblX1KOCnwNsAkuwKHAA8su3z4STrJVkP+BCwL7Ar\ncGCrC/Be4Oiq2hlYARzSyg8BVrTyo1s9SZI0pElJFKrqG8DyMWVfrqq72ur5wOy2PB84raruqKqr\ngUXA7u2zqKquqqo7gdOA+UkCPAM4o+1/MvD8gWOd3JbPAPZq9SVJ0hCmyhyFvwG+2Ja3AxYPbFvS\nyvrKtwRuGkg6Vpbf41ht+82tviRJGsLIE4Uk7wDuAk4dcRyHJlmYZOGyZctGGYokSVPGSBOFJC8H\nngu8pKqqFS8Fth+oNruV9ZXfCGyWZP0x5fc4Vtu+aav/J6rquKqaW1VzZ82atYZnJknS9DCyRCHJ\nPOAtwPOq6vaBTWcBB7Q7FnYCdgG+B1wI7NLucNiAbsLjWS3B+Bqwf9t/AXDmwLEWtOX9ga8OJCSS\nJGk11l99lTWX5OPA04CtkiwBDqe7y2FD4Nw2v/D8qnpVVV2W5HTgcrohiddU1e/bcV4LnAOsB5xY\nVZe1Jt4KnJbkSOAS4IRWfgLw0SSL6CZTHjDhJytJ0jQyKYlCVR04TvEJ45StrH8UcNQ45WcDZ49T\nfhXdXRFjy38LvPDPClaSJP3ByCczSpKkqctEQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1MlGQ\nJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMF\nSZLUy0RBkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1OteJQpJnp7k\nqWs7GEmSNLUMlSgk+XqSJ7XltwKnAf+d5O0TGZwkSRqtYXsUdgPOb8uvBJ4O7Am8apidk5yY5Pok\nPxoo2yLJuUmubN+bt/IkOSbJoiQ/TPLYgX0WtPpXJlkwUP64JJe2fY5JklW1IUmShjNsonAfoJI8\nFEhVXV5Vi4FhL7wnAfPGlB0GfKWqdgG+0tYB9gV2aZ9DgWOhu+gDhwN7ALsDhw9c+I+lS2BW7jdv\nNW1IkqQhDJsofAv4IPCvwGcAWtJwwzA7V9U3gOVjiucDJ7flk4HnD5SfUp3zgc2SbAPsA5xbVcur\nagVwLjCvbdukqs6vqgJOGXOs8dqQJElDGDZReDlwE/BD4F2t7BHAB9ag7a2r6tq2fB2wdVveDlg8\nUG9JK1tV+ZJxylfVhiRJGsL6w1SqqhuBt48p+8LaCqKqKkmtrePdmzaSHEo31MEOO+wwkaFIkrTO\nGPauhw2THJXkqiQ3t7K9k7x2Ddr+VRs2oH1f38qXAtsP1JvdylZVPnuc8lW18Seq6riqmltVc2fN\nmnWvT0qSpOlk2KGHo+nufHgJsPKv8suAv1uDts8CVt65sAA4c6D8oHb3w57AzW344Bxg7ySbt0mM\newPntG23JNmz3e1w0JhjjdeGJEkawlBDD8D/AHauql8nuRugqpYm2W41+wGQ5OPA04Ctkiyhu3vh\nPcDpSQ4BrgFe1KqfDTwbWATcDhzc2lue5N3Aha3eEVW1coLkq+nurNgI+GL7sIo2JEnSEIZNFO4c\nWzfJLODGYXauqgN7Nu01Tt0CXtNznBOBE8cpX0jX4zG2/Mbx2pAkScMZdujhk8DJSXaCP4z3f5Du\nCY2SJGmaGjZReDtwNXApsBlwJfBL4IgJikuSJE0Bw94eeSfwRuCNbcjhhjZEIEmSprFhb488KMmj\nAKpqWXsmwaOTvGxiw5MkSaM07NDDu7nnUxFp60eu3XAkSdJUMmyisAlwy5iym+nmK0iSpGlq2ETh\ncuCvx5T9D+CKtRuOJEmaSoZ9jsJbgbOTvBj4GbAz3fMJnj1RgUmSpNEbqkehqr4F/CXdUxE3Br4H\n7FZV357A2CRJ0ogN26NAVV1D90hkSZI0QwyVKCTZAvgH4K+ABwxuq6qnTEBckiRpChi2R+G/gQ2B\n0+le1CRJkmaAYROFJwKzquqOiQxGkiRNLcPeHvlDYPZEBiJJkqaeYXsUvgp8KclHgOsGN7RXP0uS\npGlo2EThycAS4FljygswUZAkaZoa9u2RT5/oQCRJ0tQz7BwFkmyZ5GVJ3tzWt03ivAVJkqaxYV8z\n/VTgJ8BLgHe24l2AYycoLkmSNAUM26PwfuDFVTUPuKuVXQDsPiFRSZKkKWHYRGFOVX2lLVf7vpM/\n4xHQkiRp3TP0a6aT7DOm7JnApWs5HkmSNIUM2yPwJuDzSb4AbJTkP4H9gPkTFpkkSRq5YXsUvgc8\nCriM7rkJVwO7V9WFExWYJEkavdX2KCRZD7gN2Kyq/mXiQ5IkSVPFansUqur3wE+BLSc+HEmSNJUM\nO0fhVLo5Ch+ge5TzyjsfqKqvTkRgkiRp9IZNFP6ufb9rTHkBD1lr0UiSpClltUMPSe4DvAJ4RFXt\nNOazxklCkjcmuSzJj5J8PMn9kuyU5IIki5J8IskGre6GbX1R2z5n4Dhva+U/GbyVM8m8VrYoyWFr\nGq8kSTPJMHMU7gY+W1V3rO3Gk2wHvB6YW1W7AesBBwDvBY6uqp2BFcAhbZdDgBWt/OhWjyS7tv0e\nCcwDPpxkvTYR80PAvsCuwIGtriRJGsKwt0d+I8meExTD+nTPZlgfuD9wLfAM4Iy2/WTg+W15flun\nbd8rSVr5aVV1R1VdDSyie7z07sCiqrqqqu4ETsNnP0iSNLRh5yhcA3wxyZnAYu45mfGdvXutRlUt\nTfKvwC+A3wBfBi4Cbqqqle+UWAJs15a3a+1TVXcluZnuboztgPMHDj24z+Ix5XuMF0uSQ4FDAXbY\nYYd7e0qSJE0rw/YobAR8li5BmA1sP/C515JsTvcX/k7AtsDGdEMHk66qjququVU1d9asWaMIQZKk\nKWeoHoWqOniC2n8mcHVVLQNI8mngScBmSdZvvQqzgaWt/lK65GRJG6rYFLhxoHylwX36yiVJ0moM\n1aOQ5CF9nzVs/xfAnknu3+Ya7AVcDnwN2L/VWQCc2ZbPauu07V+tqmrlB7S7InYCdqF77PSFwC7t\nLooN6CY8nrWGMUuSNGMMO0dhEd2wQwbKVs5TWO/eNl5VFyQ5A7gYuAu4BDgO+AJwWpIjW9kJbZcT\ngI8mWQQsp7vwU1WXJTmdLsm4C3hNe6IkSV4LnNPiPLGqLru38UqSNNMMO/Rwj56HJA8GDge+uaYB\nVNXh7ViDrqK7Y2Fs3d8CL+w5zlHAUeOUnw2cvaZxSpI0Ew07mfEequo64A3A/1674UiSpKnkXiUK\nzcPpnnsgSZKmqaGGHpJ8k4FnJ9AlCI8EjpiIoCRJ0tQw7GTG48es/xr4QVVduZbjkSRJU8iwkxlP\nXn0tSZI03Qz7HIVPJ3nymLInt1sbJUnSNDXsZManAt8ZU/Zd4OlrNxxJkjSVDJso/JbuPQyDHgD8\nbu2GI0mSppJhE4VzgP9MsglA+/4g8KWJCkySJI3esInCm4BNgOVJrqd7fPKmdA9dkiRJ09Swdz2s\nAJ7THt28PbC4PZ1RkiRNY8M+cGlv4OdV9VPgulb2cGCHqjp3AuOTJEkjNOzQw4eAW8eU3drKJUnS\nNDVsovCgqrp2TNm1wIPXcjySJGkKGTZRuCrJM8aUPQ24eu2GI0mSppJh3/XwLuDTSU4AfgY8FDi4\nfSRJ0jQ1VI9CVZ0J7E330KXntO99WrkkSZqmhu1RoKq+B3xvAmORJElTzGp7FJLMSXJSkqVJ7mjf\nJyd5yGQEKEmSRmeViUKSvwAuBh4EvAN4XvueBSxs2yVJ0jS1uqGH9wAfqqp/GlN+UpIjgX8B9puQ\nyCRJ0sitLlF4CrCgZ9u/4e2RkiRNa6ubo7Ae/a+S/l3bLkmSpqnVJQoX0v+shJcDC9dqNJIkaUpZ\n3dDDPwHntBdAnUH32OZtgBfSDUnsM7HhSZKkUVplj0JVfYfuQUuPBr4C/Lh9PxqY17ZLkqRparUP\nXKqq7wJPSbIRsAWwoqpun/DIJEnSyA37Uiiq6jdVtXRtJwlJNktyRpIfJ7kiyROSbJHk3CRXtu/N\nW90kOSbJoiQ/TPLYgeMsaPWvTLJgoPxxSS5t+xyTJGszfkmSprOhE4UJ9AHgS1X1CLohjSuAw4Cv\nVNUudEMdh7W6+wK7tM+hwLEASbYADgf2AHYHDl+ZXLQ6rxzYb94knJMkSdPCSBOFJJvSPavhBICq\nurOqbgLmAye3aicDz2/L84FTqnM+sFmSbegmVZ5bVcuragVwLjCvbdukqs6vqgJOGTiWJElajd5E\nIcn7BpafMUHt7wQsAz6S5JIkxyfZGNi6qq5tda4Dtm7L2wGLB/Zf0spWVb5knPI/keTQJAuTLFy2\nbNkanpYkSdPDqnoUDh1Y/uwEtb8+8Fjg2Kp6DPBr/jjMAEDrCagJan+wneOqam5VzZ01a9ZENydJ\n0jphVXc9/CDJGcDlwIZJjhivUlW9cw3aXwIsqaoL2voZdInCr5JsU1XXtuGD69v2pcD2A/vPbmVL\ngaeNKT+vlc8ep74kSRrCqnoU9ge+T/eApdBdoMd+ZvfuPYSqug5Y3B7oBLAXXWJyFn98x8QC4My2\nfBZwULv7YU/g5jZEcQ6wd5LN2yTGvYFz2rZbkuzZ7nY4aOBYkiRpNXp7FKrqeuBIgCTrV1Xfo5zX\n1OuAU5NsAFxF98jo+wCnJzkEuAZ4Uat7NvBsYBFwe6tLVS1P8m66R04DHFFVy9vyq4GTgI2AL7aP\nJEkawmofuARQVQe3v9T3o5sMuBT4/MDF+F6rqu8Dc8fZtNc4dQt4Tc9xTgROHKd8IbDbGoYpSdKM\nNNTtkUmeAPwMeBXwKOBvgUWtXJIkTVND9SgA7wdeXVWnrSxI8mLgGODxExGYJEkavWEfuPQw4PQx\nZWcAO6/dcCRJ0lQybKJwJXDAmLIX0g1HSJKkaWrYoYc3AJ9P8nq6uxDm0L034bkTFJckSZoChr3r\n4TtJHgo8B9gW+Bxw9tq460GSJE1dw/Yo0F629LEJjEWSJE0xU+E105IkaYoyUZAkSb1MFCRJUq+h\nE4UkO05kIJIkaer5c3oULgFot0hKkqQZYJV3PSS5CLiILklYrxW/i+7RzZIkaZpbXY/C/sCXgR2B\n+ye5GNgwydOTbDrh0UmSpJFaXaKwXlWdUVWHAbcC84EArwO+n+TKiQ5QkiSNzuoeuHRqkh2Ay4H7\nAZsDv62qFwAk2WKC45MkSSO0ykShqvZIsj7wl8C3gA8CD0xyLHBx+/gYZ0mSpqnV3vVQVXdV1SXA\nnVX1FODXwHl0L4V678SGJ0mSRmnodz0Ab2zfVVWfAD4xAfFIkqQpZOjnKFTVSW3xIRMTiiRJmmr+\n7Ec4t7dISpKkGcB3PUiSpF4mCpIkqZeJgiRJ6mWiIEmSepkoSJKkXiYKkiSp15RIFJKsl+SSJJ9v\n6zsluSDJoiSfSLJBK9+wrS9q2+cMHONtrfwnSfYZKJ/XyhYlOWyyz02SpHXZlEgUgL8HrhhYfy9w\ndFXtDKwADmnlhwArWvnRrR5JdgUOAB4JzAM+3JKP9YAPAfsCuwIHtrqSJGkII08UkswGngMc39YD\nPAM4o1U5GXh+W57f1mnb92r15wOnVdUdVXU1sAjYvX0WVdVVVXUncFqrK0mShjDyRAF4P/AW4O62\nviVwU1Xd1daXANu15e2AxdC9rAq4udX/Q/mYffrKJUnSEEaaKCR5LnB9VV00yjhaLIcmWZhk4bJl\ny0YdjiRJU8KoexSeBDwvyc/phgWeAXwA2CzJyjdbzgaWtuWlwPYAbfumwI2D5WP26Sv/E1V1XFXN\nraq5s2bNWvMzkyRpGhhpolBVb6uq2VU1h24y4ler6iXA14D9W7UFwJlt+ay2Ttv+1aqqVn5Auyti\nJ2AX4HvAhcAu7S6KDVobZ03CqUmSNC2sv/oqI/FW4LQkRwKXACe08hOAjyZZBCynu/BTVZclOR24\nHLgLeE1V/R4gyWuBc4D1gBOr6rJJPRNJktZhUyZRqKrzgPPa8lV0dyyMrfNb4IU9+x8FHDVO+dnA\n2WsxVEmSZoxRz1GQJElTmImCJEnqZaIgSZJ6mShIkqReJgqSJKmXiYIkSeploiBJknqZKEiSpF4m\nCpIkqZeJgiRJ6mWiIEmSepkoSJKkXiYKkiSpl4mCJEnqZaIgSZJ6mShIkqReJgqSJKmXiYIkSepl\noiBJknqZKEiSpF4mCpIkqZeJgiRJ6mWiIEmSepkoSJKkXiYKkiSpl4mCJEnqZaIgSZJ6jTRRSLJ9\nkq8luTzJZUn+vpVvkeTcJFe2781beZIck2RRkh8meezAsRa0+lcmWTBQ/rgkl7Z9jkmSyT9TSZLW\nTaPuUbgLeFNV7QrsCbwmya7AYcBXqmoX4CttHWBfYJf2ORQ4FrrEAjgc2APYHTh8ZXLR6rxyYL95\nk3BekiRNCyNNFKrq2qq6uC3fClwBbAfMB05u1U4Gnt+W5wOnVOd8YLMk2wD7AOdW1fKqWgGcC8xr\n2zapqvOrqoBTBo4lSZJWY9Q9Cn+QZA7wGOACYOuqurZtug7Yui1vBywe2G1JK1tV+ZJxyiVJ0hCm\nRKKQ5AHAp4A3VNUtg9taT0BNQgyHJlmYZOGyZcsmujlJktYJI08UktyXLkk4tao+3Yp/1YYNaN/X\nt/KlwPYDu89uZasqnz1O+Z+oquOqam5VzZ01a9aanZQkSdPEqO96CHACcEVV/fvAprOAlXcuLADO\nHCg/qN39sCdwcxuiOAfYO8nmbRLj3sA5bdstSfZsbR00cCxJkrQa64+4/ScBLwMuTfL9VvZ24D3A\n6UkOAa4BXtS2nQ08G1gE3A4cDFBVy5O8G7iw1Tuiqpa35VcDJwEbAV9sH0mSNISRJgpV9S2g77kG\ne41Tv4DX9BzrRODEccoXArutQZiSJM1YI5+jIEmSpi4TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMF\nSZLUy0RBkiT1MlGQJEm9TFvIugkAAAlESURBVBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8T\nBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1Wn/UAWjd8Ysj/nLUIaw1O7zz0lGHIEnr\nBHsUJElSLxMFSZLUy0RBkiT1co6CpKF8/SlPHXUIa81Tv/H1UYcgrTPsUZAkSb3sUZCG9KT/86RR\nh7DWfPt13x51CJLWEfYoSJKkXjMiUUgyL8lPkixKctio45EkaV0x7YcekqwHfAh4FrAEuDDJWVV1\n+Wgjk7Su+OCbPjfqENaa1/7bfqMOQeuYmdCjsDuwqKquqqo7gdOA+SOOSZKkdcK071EAtgMWD6wv\nAfYYUSyStM456qX7jzqEteYdHztj1CGsc1JVo45hQiXZH5hXVa9o6y8D9qiq146pdyhwaFt9OPCT\nSQ30nrYCbhhh+6M2k89/Jp87eP6e/8w9/1Gf+45VNWu8DTOhR2EpsP3A+uxWdg9VdRxw3GQFtSpJ\nFlbV3FHHMSoz+fxn8rmD5+/5z9zzn8rnPhPmKFwI7JJkpyQbAAcAZ404JkmS1gnTvkehqu5K8lrg\nHGA94MSqumzEYUmStE6Y9okCQFWdDZw96jj+DFNiCGSEZvL5z+RzB8/f85+5puy5T/vJjJIk6d6b\nCXMUJEnSvWSiMIXM9EdNJzkxyfVJfjTqWCZbku2TfC3J5UkuS/L3o45pMiW5X5LvJflBO/9/HnVM\nky3JekkuSfL5Uccy2ZL8PMmlSb6fZOGo45lsSTZLckaSHye5IskTRh3TIIcepoj2qOmfMvCoaeDA\nmfSo6SRPAW4DTqmq3UYdz2RKsg2wTVVdnOSBwEXA82fK7z9JgI2r6rYk9wW+Bfx9VZ0/4tAmTZL/\nH5gLbFJVzx11PJMpyc+BuVU1I5+hkORk4JtVdXy7O+/+VXXTqONayR6FqWPGP2q6qr4BLB91HKNQ\nVddW1cVt+VbgCrqnis4I1bmtrd63fWbMXzFJZgPPAY4fdSyaXEk2BZ4CnABQVXdOpSQBTBSmkvEe\nNT1jLhT6oyRzgMcAF4w2ksnVut6/D1wPnFtVM+n83w+8Bbh71IGMSAFfTnJRe0ruTLITsAz4SBt6\nOj7JxqMOapCJgjSFJHkA8CngDVV1y6jjmUxV9fuq+iu6p6funmRGDD8leS5wfVVdNOpYRuj/q6rH\nAvsCr2nDkDPF+sBjgWOr6jHAr4EpNUfNRGHqGOpR05q+2tj8p4BTq+rTo45nVFq369eAeaOOZZI8\nCXheG6c/DXhGko+NNqTJVVVL2/f1wGfohmJniiXAkoEetDPoEocpw0Rh6vBR0zNYm8x3AnBFVf37\nqOOZbElmJdmsLW9EN6n3x6ONanJU1duqanZVzaH77/6rVfXSEYc1aZJs3Cbw0rrc9wZmzJ1PVXUd\nsDjJw1vRXsCUmsQ8I57MuC7wUdOQ5OPA04CtkiwBDq+qE0Yb1aR5EvAy4NI2Tg/w9vZU0ZlgG+Dk\ndvfPfYDTq2rG3SY4Q20NfKbLlVkf+O+q+tJoQ5p0rwNObX8kXgUcPOJ47sHbIyVJUi+HHiRJUi8T\nBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSUNJsnWSbyS5Ncm/jTqetS3JF5MsGHUc0lRjoiBNc+31\nzQ9L8pAkF6/BoQ4FbqB7u+GbetraPcnZSW5Ksry1PdQ94UnOS/KKNYhvjVTVvlV18qjal6YqEwVp\nGmuPhd4RuBJ4HLAmicKOwOXV8/CVJE8Avgp8HdgZ2BL4O7rn909Z6fj/QqmH/3FI09tu/PHiPpfV\nJApJnpjkwiQ3t+8ntvKTgAXAW5LcluSZ4+z+PuDkqnpvVd3QXh19UVW9qB1j8ySfT7IsyYq2PLtt\nOwp4MvDBdvwPtvJHJDm39U78JMmLBmLdMsnnktzSYj0yybdWdy5t23lJjkrybeB24CFjezSS/E2S\nK1qs5yTZsZUnydFJrm9tXzpTXmClGaqq/PjxM80+dI+AvYnuIvjbtnwXcGtb3mmcfbYAVtA9Snp9\n4MC2vmXbfhJwZE979wd+Dzx9FTFtCfx1q/tA4JPAZwe2nwe8YmB9Y7pXrx/c4nkM3dDHrm37ae1z\nf2DXVvdbQ57LecAvgEe27fcdbB+YDywC/qJt/0fgO23bPsBFwGZAWp1tRv079+Nnoj72KEjTUFV9\npKo2o7ug7Qk8iu5FO5tU1WZVdfU4uz0HuLKqPlpVd1XVx+lezLTfEE1uTtdDee0qYrqxqj5VVbdX\n1a3AUcBTV3HM5wI/b+dyV1VdQvd2zRe2d0L8Nd37QG6vqsuBwfkFw5zLSVV1Wdv+uzFtvwr431V1\nRVXdBfwv4K9ar8Lv6BKdR9A9Bv+Kquo9b2ldZ6IgTTNJtmiTCW8Gnkj3l/JPgIcDK5K8oWfXbYFr\nxpRdA2w3RLMrgLvpXu7UF9f9k/xnkmuS3AJ8A9isXfTHsyOwRzuXm5LcBLwEeDAwi+4v/cUD9QeX\nhzmXxfTbEfjAQLvL6XoPtquqrwIfBD4EXJ/kuCSbrOJY0jrNREGaZqpqeetN+Fvg+Lb8JWC/1pvw\n/p5df0l3gRy0A7B0iDZvB75L91d+nzfRJSt7VNUmwFNaeVYeZkz9xcDXW8wrPw+oqr8DltENpcwe\nqL/9n3kuq3oj3mLgb8e0vVFVfaed7zFV9Ti6IY+HAW9exbGkdZqJgjR9Dd7l8Bi6YYhVORt4WJL/\nmWT9JC+muxAO+7rntwAvT/LmJFsCJHl0ktPa9gcCvwFuSrIFcPiY/X8FPGRg/fMtnpcluW/7PD7J\nX1TV74FPA+9qPRWPAA5ai+fyH8DbkjyyncemSV7Ylh+fZI92R8mv6eaA3D3kcaV1jomCNH09Dri4\nXbR/X1UrVlW5qm6kmxfwJuBGugv/c6vqhmEaa39tP6N9rkqyHDiO7qIN8H5gI7oJiefT9XIM+gCw\nf7vL4Jg2j2Fv4AC6HoLrgPcCG7b6rwU2beUfBT4O3LGWzuUzra3T2jDJj/jjbZ6bAP9FN9xyTTv+\n+4Y5rrQuStWqet8kad2Q5L3Ag6vKpytKa5E9CpLWSe0ZC49qzzXYHTgE+Myo45Kmm/VHHYAk3UsP\npBtu2JZufsO/AWeONCJpGnLoQZIk9XLoQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1MlGQJEm9\n/h/c0EptfROsNQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 576x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xA0VkBh5sJOW",
        "outputId": "8d4f6147-2770-4b0a-d606-e94b20fc2b4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# How many comments are unlabeled\n",
        "print('Percentage of comments that are not labelled:')\n",
        "print(len(df[(df['toxic']==0) & (df['severe_toxic']==0) & (df['obscene']==0) & (df['threat']== 0) & (df['insult']==0) & (df['identity_hate']==0)]) / len(df))\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Percentage of comments that are not labelled:\n",
            "0.8983211235124177\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7w7NT134sN-g",
        "outputId": "122c7a58-f4d4-4ad3-8c28-c483c0e178b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Function for cleaning the comment text\n",
        "def clean_text(text):\n",
        "    text = text.lower()\n",
        "    text = re.sub(r\"what's\", \"what is \", text)\n",
        "    text = re.sub(r\"\\'s\", \" \", text)\n",
        "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
        "    text = re.sub(r\"can't\", \"can not \", text)\n",
        "    text = re.sub(r\"n't\", \" not \", text)\n",
        "    text = re.sub(r\"i'm\", \"i am \", text)\n",
        "    text = re.sub(r\"\\'re\", \" are \", text)\n",
        "    text = re.sub(r\"\\'d\", \" would \", text)\n",
        "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
        "    text = re.sub(r\"\\'scuse\", \" excuse \", text)\n",
        "    text = re.sub('\\W', ' ', text)\n",
        "    text = re.sub('\\s+', ' ', text)\n",
        "    text = text.strip(' ')\n",
        "    return text\n",
        "\n",
        "# Use the helper function to clean the text\n",
        "df['comment_text'] = df['comment_text'].map(lambda comment : clean_text(comment))\n",
        "# print(df['comment_text'][0])\n",
        "\n",
        "# Splitting the data into train and test sets\n",
        "categories = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
        "train, test = train_test_split(df, random_state=42, test_size=0.33, shuffle=True)\n",
        "X_train = train.comment_text\n",
        "X_test = test.comment_text\n",
        "print('train shape is', X_train.shape)\n",
        "print('test shape is', X_test.shape)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train shape is (106912,)\n",
            "test shape is (52659,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "g1VmWCuusV1u",
        "outputId": "3dc34714-fce9-46e4-a02e-8ced7841f7cb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# from sklearn.utils.class_weight import compute_class_weight\n",
        "# class_weights = compute_class_weight('balanced', np.unique(test), test)\n",
        "\n",
        "# Define a pipeline combining a text feature extractor with multi lable classifier\n",
        "# 1. Naive Bayes Pipeline\n",
        "#    This is essentially a Naive Bayes, but we pass the NB model into a OneVsRestClassifier,\n",
        "#    which is one of the methods we can use for datasets with multiple labels\n",
        "print('Training Naive Bayes Pipeline\\n')\n",
        "NB_pipeline = Pipeline([\n",
        "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
        "                ('clf', OneVsRestClassifier(MultinomialNB(\n",
        "                    fit_prior=True, class_prior=None))),\n",
        "            ])\n",
        "for category in categories:\n",
        "    print('--------- Training {} ---------\\n'.format(category))\n",
        "    # train the model using X_dtm & y\n",
        "    NB_pipeline.fit(X_train, train[category])\n",
        "    # compute the testing accuracy\n",
        "    prediction = NB_pipeline.predict(X_test)\n",
        "    print(classification_report(test[category], prediction, labels=np.unique(test[category])))\n",
        "#     print(prediction.head())\n",
        "#     print(prediction)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Naive Bayes Pipeline\n",
            "\n",
            "--------- Training toxic ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.92      1.00      0.96     47576\n",
            "           1       0.99      0.16      0.28      5083\n",
            "\n",
            "    accuracy                           0.92     52659\n",
            "   macro avg       0.95      0.58      0.62     52659\n",
            "weighted avg       0.93      0.92      0.89     52659\n",
            "\n",
            "--------- Training severe_toxic ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      1.00      0.99     52133\n",
            "           1       0.00      0.00      0.00       526\n",
            "\n",
            "    accuracy                           0.99     52659\n",
            "   macro avg       0.50      0.50      0.50     52659\n",
            "weighted avg       0.98      0.99      0.99     52659\n",
            "\n",
            "--------- Training obscene ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      1.00      0.98     49828\n",
            "           1       0.99      0.10      0.18      2831\n",
            "\n",
            "    accuracy                           0.95     52659\n",
            "   macro avg       0.97      0.55      0.58     52659\n",
            "weighted avg       0.95      0.95      0.93     52659\n",
            "\n",
            "--------- Training threat ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      1.00      1.00     52507\n",
            "           1       0.00      0.00      0.00       152\n",
            "\n",
            "    accuracy                           1.00     52659\n",
            "   macro avg       0.50      0.50      0.50     52659\n",
            "weighted avg       0.99      1.00      1.00     52659\n",
            "\n",
            "--------- Training insult ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      1.00      0.98     50016\n",
            "           1       0.98      0.04      0.07      2643\n",
            "\n",
            "    accuracy                           0.95     52659\n",
            "   macro avg       0.97      0.52      0.53     52659\n",
            "weighted avg       0.95      0.95      0.93     52659\n",
            "\n",
            "--------- Training identity_hate ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      1.00      1.00     52188\n",
            "           1       0.00      0.00      0.00       471\n",
            "\n",
            "    accuracy                           0.99     52659\n",
            "   macro avg       0.50      0.50      0.50     52659\n",
            "weighted avg       0.98      0.99      0.99     52659\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "x6C0npvhsYO4",
        "outputId": "d1040e82-6c5d-461f-e508-9985227d1861",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Linear SVC\n",
        "#    This is essentially an SVM, but we pass the SVM into a OneVsRestClassifier,\n",
        "#    which is one of the methods we can use for datasets with multiple labels\n",
        "print('Training Linear SVC Pipeline\\n')\n",
        "SVC_pipeline = Pipeline([\n",
        "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
        "                ('clf', OneVsRestClassifier(LinearSVC(max_iter = 400, class_weight='balanced'), n_jobs=1)),\n",
        "            ])\n",
        "for category in categories:\n",
        "    print('--------- Training {} ---------\\n'.format(category))\n",
        "    # train the model using X_dtm & y\n",
        "    SVC_pipeline.fit(X_train, train[category])\n",
        "    # compute the testing accuracy\n",
        "    prediction = SVC_pipeline.predict(X_test)\n",
        "#     print('... ... Test accuracy is {}'.format(accuracy_score(test[category], prediction)))\n",
        "    print(classification_report(test[category], prediction, labels=np.unique(test[category])))\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Linear SVC Pipeline\n",
            "\n",
            "--------- Training toxic ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.97      0.97     47576\n",
            "           1       0.71      0.80      0.75      5083\n",
            "\n",
            "    accuracy                           0.95     52659\n",
            "   macro avg       0.84      0.88      0.86     52659\n",
            "weighted avg       0.95      0.95      0.95     52659\n",
            "\n",
            "--------- Training severe_toxic ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.99      0.99     52133\n",
            "           1       0.33      0.61      0.43       526\n",
            "\n",
            "    accuracy                           0.98     52659\n",
            "   macro avg       0.66      0.80      0.71     52659\n",
            "weighted avg       0.99      0.98      0.99     52659\n",
            "\n",
            "--------- Training obscene ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.99      0.99     49828\n",
            "           1       0.76      0.82      0.79      2831\n",
            "\n",
            "    accuracy                           0.98     52659\n",
            "   macro avg       0.88      0.90      0.89     52659\n",
            "weighted avg       0.98      0.98      0.98     52659\n",
            "\n",
            "--------- Training threat ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      1.00      1.00     52507\n",
            "           1       0.41      0.47      0.44       152\n",
            "\n",
            "    accuracy                           1.00     52659\n",
            "   macro avg       0.70      0.74      0.72     52659\n",
            "weighted avg       1.00      1.00      1.00     52659\n",
            "\n",
            "--------- Training insult ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.98      0.98     50016\n",
            "           1       0.62      0.76      0.69      2643\n",
            "\n",
            "    accuracy                           0.97     52659\n",
            "   macro avg       0.81      0.87      0.83     52659\n",
            "weighted avg       0.97      0.97      0.97     52659\n",
            "\n",
            "--------- Training identity_hate ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.99      0.99     52188\n",
            "           1       0.39      0.54      0.45       471\n",
            "\n",
            "    accuracy                           0.99     52659\n",
            "   macro avg       0.69      0.76      0.72     52659\n",
            "weighted avg       0.99      0.99      0.99     52659\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MAxUJ92Kseq1",
        "outputId": "2519e0a6-9d59-4bf5-bf7d-778b44e0983d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# Logistic Regression Pipeline\n",
        "#    This is essentially a Logistic Regression model, but we pass the Logistic Regression into a OneVsRestClassifier,\n",
        "#    which is one of the methods we can use for datasets with multiple labels\n",
        "print('Training Logistic Regression Pipeline\\n')\n",
        "LogReg_pipeline = Pipeline([\n",
        "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
        "                ('clf', OneVsRestClassifier(LogisticRegression(max_iter = 400, solver='sag', class_weight='balanced'), n_jobs=1)),\n",
        "            ])\n",
        "for category in categories:\n",
        "    print('--------- Training {} ---------\\n'.format(category))\n",
        "    # train the model using X_dtm & y\n",
        "    LogReg_pipeline.fit(X_train, train[category])\n",
        "    # compute the testing accuracy\n",
        "    prediction = LogReg_pipeline.predict(X_test)\n",
        "    print(classification_report(test[category], prediction, labels=np.unique(test[category])))\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Logistic Regression Pipeline\n",
            "\n",
            "--------- Training toxic ---------\n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.98      0.95      0.97     47576\n",
            "           1       0.66      0.85      0.74      5083\n",
            "\n",
            "    accuracy                           0.94     52659\n",
            "   macro avg       0.82      0.90      0.85     52659\n",
            "weighted avg       0.95      0.94      0.95     52659\n",
            "\n",
            "--------- Training severe_toxic ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.95      0.97     52133\n",
            "           1       0.14      0.87      0.25       526\n",
            "\n",
            "    accuracy                           0.95     52659\n",
            "   macro avg       0.57      0.91      0.61     52659\n",
            "weighted avg       0.99      0.95      0.97     52659\n",
            "\n",
            "--------- Training obscene ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.98      0.99     49828\n",
            "           1       0.71      0.86      0.78      2831\n",
            "\n",
            "    accuracy                           0.97     52659\n",
            "   macro avg       0.85      0.92      0.88     52659\n",
            "weighted avg       0.98      0.97      0.97     52659\n",
            "\n",
            "--------- Training threat ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.93      0.97     52507\n",
            "           1       0.04      0.86      0.07       152\n",
            "\n",
            "    accuracy                           0.93     52659\n",
            "   macro avg       0.52      0.89      0.52     52659\n",
            "weighted avg       1.00      0.93      0.96     52659\n",
            "\n",
            "--------- Training insult ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.97      0.98     50016\n",
            "           1       0.56      0.84      0.67      2643\n",
            "\n",
            "    accuracy                           0.96     52659\n",
            "   macro avg       0.78      0.90      0.83     52659\n",
            "weighted avg       0.97      0.96      0.96     52659\n",
            "\n",
            "--------- Training identity_hate ---------\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.92      0.96     52188\n",
            "           1       0.08      0.85      0.15       471\n",
            "\n",
            "    accuracy                           0.92     52659\n",
            "   macro avg       0.54      0.88      0.55     52659\n",
            "weighted avg       0.99      0.92      0.95     52659\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}