taxes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Investigating tax topic to see if it can be further divided into subtopics  \n",
    "Worried it may be a catch-all topic because of its size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "doj = pd.read_csv('topics.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxes = doj[doj['topicname']=='Taxes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1503 entries, 58 to 13061\n",
      "Data columns (total 9 columns):\n",
      "Unnamed: 0         1503 non-null int64\n",
      "components         1503 non-null object\n",
      "contents           1503 non-null object\n",
      "date               1503 non-null object\n",
      "title              1503 non-null object\n",
      "topicnumber        1503 non-null int64\n",
      "strengthoftopic    1503 non-null float64\n",
      "year               1503 non-null int64\n",
      "topicname          1503 non-null object\n",
      "dtypes: float64(1), int64(3), object(5)\n",
      "memory usage: 117.4+ KB\n"
     ]
    }
   ],
   "source": [
    "taxes.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizing and transforming contents of articles in tax subtopic only to create an NMF matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "import numpy as np\n",
    "from sklearn.decomposition import NMF\n",
    "import string\n",
    "import unidecode\n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "import re\n",
    "\n",
    "def customtokenizer(article):\n",
    "    punc = str.maketrans('','',string.punctuation+\"''``''``\\\"\")\n",
    "    article_c = article.translate(punc)\n",
    "    dig = str.maketrans('','',string.digits)\n",
    "    article_c=article_c.translate(dig)\n",
    "    article_c = unidecode.unidecode(article_c)\n",
    "    article_c = article_c.lower()\n",
    "    regex = re.compile(r'(?u)\\b\\w\\w+\\b')\n",
    "    article_c = re.findall(regex,article_c)\n",
    "    stop_words = stopwords.words('english')\n",
    "    article_c = [y for y in article_c if y not in stop_words]\n",
    "    stemmer = SnowballStemmer('english')\n",
    "    article_c = [stemmer.stem(y) for y in article_c] \n",
    "    return article_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(tokenizer=customtokenizer, stop_words=stopwords.words('english'))\n",
    " \n",
    "X = vectorizer.fit_transform(taxes['contents'])\n",
    " \n",
    "idx_to_word = np.array(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Topic 1: deduct,ir,one,perman,enjoin,court,incom,justic,depart,claim,busi,feder,credit,alleg,injunct,complaint,custom,return,prepar,tax\n",
      "Topic 2: incom,special,trial,agent,addit,district,fals,year,file,ciraolo,prison,deputi,act,general,sentenc,us,divis,investig,assist,attorney\n",
      "Topic 3: time,million,respons,depart,cash,busi,wage,quarter,court,account,collect,payrol,withheld,employe,compani,ir,fail,pay,employ,tax\n",
      "Topic 4: defend,year,beyond,mere,incom,obstruct,file,juri,maximum,alleg,grand,proven,innoc,presum,convict,ir,fals,charg,count,indict\n",
      "Topic 5: court,jenkin,bogus,larg,frivol,marti,issu,promot,fals,million,withhold,request,base,redempt,fraudul,claim,oid,scheme,form,refund\n"
     ]
    }
   ],
   "source": [
    "nmf = NMF(n_components=5, solver=\"mu\")\n",
    " \n",
    "W = nmf.fit_transform(X)\n",
    " \n",
    "H = nmf.components_\n",
    " \n",
    "# print the topics\n",
    " \n",
    "for i, topic in enumerate(H):\n",
    " \n",
    "    print(\"Topic {}: {}\".format(i + 1, \",\".join([str(x) for x in idx_to_word[topic.argsort()[-20:]]])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### It's difficult to tell whether these topics are actually different or just spurious. It's possible that the Department of Justice just investigates and prosecutes a lot of tax crimes. Without further information about possible ways to proceed, we will continue with having taxes as just one topic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}