w2v.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating word vectors to investigate relationships between words to see if there is potential bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "import pickle\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "# gensim\n",
    "from gensim import corpora, models, similarities, matutils\n",
    "# sklearn\n",
    "from sklearn import datasets\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "import sklearn.metrics.pairwise as smp\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from sklearn.decomposition import NMF\n",
    "\n",
    "# logging for gensim (set to INFO)\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "with open('doj_c.pkl','rb') as f:\n",
    "    doj = pickle.load(f)\n",
    "data = doj['contents']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = ''\n",
    "for each in data:\n",
    "    corpus += each.lower()\n",
    "    \n",
    "import string\n",
    "import unidecode\n",
    "corpus = unidecode.unidecode(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = sent_tokenize(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "phrases = []\n",
    "for each in sentences:\n",
    "    phrases.append(word_tokenize(each))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-11-16 21:27:01,281 : INFO : collecting all words and their counts\n",
      "2018-11-16 21:27:01,282 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2018-11-16 21:27:01,356 : INFO : PROGRESS: at sentence #10000, processed 339800 words, keeping 15231 word types\n",
      "2018-11-16 21:27:01,436 : INFO : PROGRESS: at sentence #20000, processed 695328 words, keeping 23550 word types\n",
      "2018-11-16 21:27:01,516 : INFO : PROGRESS: at sentence #30000, processed 1041117 words, keeping 28567 word types\n",
      "2018-11-16 21:27:01,596 : INFO : PROGRESS: at sentence #40000, processed 1388818 words, keeping 33211 word types\n",
      "2018-11-16 21:27:01,676 : INFO : PROGRESS: at sentence #50000, processed 1736964 words, keeping 37514 word types\n",
      "2018-11-16 21:27:01,762 : INFO : PROGRESS: at sentence #60000, processed 2082734 words, keeping 41974 word types\n",
      "2018-11-16 21:27:01,843 : INFO : PROGRESS: at sentence #70000, processed 2430141 words, keeping 44988 word types\n",
      "2018-11-16 21:27:01,925 : INFO : PROGRESS: at sentence #80000, processed 2778136 words, keeping 47213 word types\n",
      "2018-11-16 21:27:02,006 : INFO : PROGRESS: at sentence #90000, processed 3126126 words, keeping 49229 word types\n",
      "2018-11-16 21:27:02,088 : INFO : PROGRESS: at sentence #100000, processed 3473401 words, keeping 52032 word types\n",
      "2018-11-16 21:27:02,168 : INFO : PROGRESS: at sentence #110000, processed 3819374 words, keeping 54320 word types\n",
      "2018-11-16 21:27:02,250 : INFO : PROGRESS: at sentence #120000, processed 4166549 words, keeping 56836 word types\n",
      "2018-11-16 21:27:02,331 : INFO : PROGRESS: at sentence #130000, processed 4518135 words, keeping 58738 word types\n",
      "2018-11-16 21:27:02,411 : INFO : PROGRESS: at sentence #140000, processed 4873982 words, keeping 60689 word types\n",
      "2018-11-16 21:27:02,489 : INFO : PROGRESS: at sentence #150000, processed 5217545 words, keeping 62788 word types\n",
      "2018-11-16 21:27:02,568 : INFO : PROGRESS: at sentence #160000, processed 5564867 words, keeping 64502 word types\n",
      "2018-11-16 21:27:02,646 : INFO : PROGRESS: at sentence #170000, processed 5904095 words, keeping 66396 word types\n",
      "2018-11-16 21:27:02,726 : INFO : PROGRESS: at sentence #180000, processed 6249110 words, keeping 68355 word types\n",
      "2018-11-16 21:27:02,815 : INFO : PROGRESS: at sentence #190000, processed 6651478 words, keeping 70591 word types\n",
      "2018-11-16 21:27:02,896 : INFO : PROGRESS: at sentence #200000, processed 7001618 words, keeping 72628 word types\n",
      "2018-11-16 21:27:02,975 : INFO : PROGRESS: at sentence #210000, processed 7338273 words, keeping 74306 word types\n",
      "2018-11-16 21:27:03,058 : INFO : PROGRESS: at sentence #220000, processed 7685738 words, keeping 75960 word types\n",
      "2018-11-16 21:27:03,140 : INFO : PROGRESS: at sentence #230000, processed 8031194 words, keeping 77375 word types\n",
      "2018-11-16 21:27:03,222 : INFO : PROGRESS: at sentence #240000, processed 8374176 words, keeping 79145 word types\n",
      "2018-11-16 21:27:03,256 : INFO : collected 79732 word types from a corpus of 8513205 raw words and 244062 sentences\n",
      "2018-11-16 21:27:03,257 : INFO : Loading a fresh vocabulary\n",
      "2018-11-16 21:27:03,379 : INFO : min_count=5 retains 30979 unique words (38% of original 79732, drops 48753)\n",
      "2018-11-16 21:27:03,380 : INFO : min_count=5 leaves 8432265 word corpus (99% of original 8513205, drops 80940)\n",
      "2018-11-16 21:27:03,698 : INFO : deleting the raw counts dictionary of 79732 items\n",
      "2018-11-16 21:27:03,702 : INFO : sample=0.001 downsamples 39 most-common words\n",
      "2018-11-16 21:27:03,703 : INFO : downsampling leaves estimated 6219465 word corpus (73.8% of prior 8432265)\n",
      "2018-11-16 21:27:03,844 : INFO : estimated required memory for 30979 words and 100 dimensions: 40272700 bytes\n",
      "2018-11-16 21:27:03,846 : INFO : resetting layer weights\n",
      "2018-11-16 21:27:04,541 : INFO : training model with 4 workers on 30979 vocabulary and 100 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
      "2018-11-16 21:27:05,588 : INFO : EPOCH 1 - PROGRESS: at 18.13% examples, 1119099 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:06,588 : INFO : EPOCH 1 - PROGRESS: at 39.02% examples, 1206016 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:07,592 : INFO : EPOCH 1 - PROGRESS: at 59.57% examples, 1229162 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:08,594 : INFO : EPOCH 1 - PROGRESS: at 77.40% examples, 1207760 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:09,599 : INFO : EPOCH 1 - PROGRESS: at 95.69% examples, 1188397 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:09,814 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-11-16 21:27:09,816 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-11-16 21:27:09,820 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-11-16 21:27:09,824 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-11-16 21:27:09,825 : INFO : EPOCH - 1 : training on 8513205 raw words (6219957 effective words) took 5.2s, 1187510 effective words/s\n",
      "2018-11-16 21:27:10,834 : INFO : EPOCH 2 - PROGRESS: at 19.87% examples, 1227918 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:11,836 : INFO : EPOCH 2 - PROGRESS: at 41.67% examples, 1284721 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:12,837 : INFO : EPOCH 2 - PROGRESS: at 63.36% examples, 1306939 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:13,838 : INFO : EPOCH 2 - PROGRESS: at 83.56% examples, 1302402 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:14,602 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-11-16 21:27:14,604 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-11-16 21:27:14,606 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-11-16 21:27:14,615 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-11-16 21:27:14,616 : INFO : EPOCH - 2 : training on 8513205 raw words (6219957 effective words) took 4.8s, 1300222 effective words/s\n",
      "2018-11-16 21:27:15,626 : INFO : EPOCH 3 - PROGRESS: at 20.47% examples, 1266096 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:16,632 : INFO : EPOCH 3 - PROGRESS: at 41.90% examples, 1289984 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:17,634 : INFO : EPOCH 3 - PROGRESS: at 61.59% examples, 1268965 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:18,635 : INFO : EPOCH 3 - PROGRESS: at 79.43% examples, 1237664 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:19,641 : INFO : EPOCH 3 - PROGRESS: at 97.94% examples, 1215096 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:19,742 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-11-16 21:27:19,746 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-11-16 21:27:19,749 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-11-16 21:27:19,754 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-11-16 21:27:19,755 : INFO : EPOCH - 3 : training on 8513205 raw words (6219680 effective words) took 5.1s, 1212509 effective words/s\n",
      "2018-11-16 21:27:20,767 : INFO : EPOCH 4 - PROGRESS: at 18.46% examples, 1137311 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:21,769 : INFO : EPOCH 4 - PROGRESS: at 39.61% examples, 1221325 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:22,771 : INFO : EPOCH 4 - PROGRESS: at 57.66% examples, 1189001 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:23,774 : INFO : EPOCH 4 - PROGRESS: at 80.13% examples, 1248102 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:24,776 : INFO : EPOCH 4 - PROGRESS: at 98.52% examples, 1222830 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:24,827 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-11-16 21:27:24,830 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-11-16 21:27:24,833 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-11-16 21:27:24,837 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-11-16 21:27:24,838 : INFO : EPOCH - 4 : training on 8513205 raw words (6219151 effective words) took 5.1s, 1225596 effective words/s\n",
      "2018-11-16 21:27:25,849 : INFO : EPOCH 5 - PROGRESS: at 20.70% examples, 1274834 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:26,854 : INFO : EPOCH 5 - PROGRESS: at 39.37% examples, 1211453 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:27,860 : INFO : EPOCH 5 - PROGRESS: at 57.30% examples, 1178783 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:28,860 : INFO : EPOCH 5 - PROGRESS: at 75.46% examples, 1162479 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:29,863 : INFO : EPOCH 5 - PROGRESS: at 96.98% examples, 1202459 words/s, in_qsize 7, out_qsize 0\n",
      "2018-11-16 21:27:29,977 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2018-11-16 21:27:29,979 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2018-11-16 21:27:29,983 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2018-11-16 21:27:29,987 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2018-11-16 21:27:29,988 : INFO : EPOCH - 5 : training on 8513205 raw words (6218879 effective words) took 5.1s, 1209172 effective words/s\n",
      "2018-11-16 21:27:29,989 : INFO : training on a 42566025 raw words (31097624 effective words) took 25.4s, 1222028 effective words/s\n"
     ]
    }
   ],
   "source": [
    "doj_vecs = models.word2vec.Word2Vec(phrases, size=100, window=5, min_count=5, workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing words related to Muslims and Islam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "2018-11-16 21:27:29,995 : INFO : precomputing L2-norms of word weight vectors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('sword', 0.6522782444953918),\n",
       " ('crusaders', 0.6412355303764343),\n",
       " ('muslims', 0.6121676564216614),\n",
       " ('infidels', 0.5829067826271057),\n",
       " ('hatred', 0.5803992748260498),\n",
       " ('backdrop', 0.5726880431175232),\n",
       " ('prejudice', 0.5712781548500061),\n",
       " ('retaliate', 0.5708475112915039),\n",
       " ('intolerance', 0.5645120739936829),\n",
       " ('enemies', 0.5580153465270996)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('islam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('somali', 0.6999996304512024),\n",
       " ('jewish', 0.6500884294509888),\n",
       " ('latino', 0.6450031995773315),\n",
       " ('native', 0.639795184135437),\n",
       " ('sikh', 0.6370640993118286),\n",
       " ('african', 0.6262637376785278),\n",
       " ('amish', 0.6157890558242798),\n",
       " ('pregnant', 0.6096385717391968),\n",
       " ('gay', 0.607383668422699),\n",
       " ('young', 0.5889614820480347)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('muslim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('motel', 0.7222611308097839),\n",
       " ('entrance', 0.6853206157684326),\n",
       " ('church', 0.6730661392211914),\n",
       " ('shotgun', 0.6614291667938232),\n",
       " ('bombs', 0.6608598232269287),\n",
       " ('restaurant', 0.6592603325843811),\n",
       " ('synagogue', 0.6552081108093262),\n",
       " ('shoot', 0.6549220681190491),\n",
       " ('gunpoint', 0.6405906081199646),\n",
       " ('location', 0.6367930173873901)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('mosque')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('me', 0.7312315106391907),\n",
       " ('you', 0.6976401209831238),\n",
       " ('...', 0.6827571988105774),\n",
       " ('something', 0.6817971467971802),\n",
       " ('die', 0.6699077486991882),\n",
       " ('let', 0.6330971717834473),\n",
       " ('[', 0.6289156675338745),\n",
       " (\"'m\", 0.6286628246307373),\n",
       " ('see', 0.6107150912284851),\n",
       " ('speak', 0.5957574844360352)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('allah')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing with words in corpus that exist for other religions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('macedonia', 0.8161101341247559),\n",
       " ('god', 0.6900001168251038),\n",
       " ('church', 0.6628494262695312),\n",
       " ('walnut', 0.6463581919670105),\n",
       " ('boulder', 0.6450909972190857),\n",
       " ('glade', 0.6440331935882568),\n",
       " ('conneaut', 0.6425847411155701),\n",
       " ('reverend', 0.6404812335968018),\n",
       " ('tuscaloosa', 0.6396113634109497),\n",
       " ('highland', 0.6354373693466187)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('christ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('congregation', 0.6941131353378296),\n",
       " ('synagogue', 0.6867195963859558),\n",
       " ('mosque', 0.6730661392211914),\n",
       " ('christ', 0.6628494262695312),\n",
       " ('motel', 0.6553056240081787),\n",
       " ('neighborhood', 0.6517730951309204),\n",
       " ('park', 0.6368703246116638),\n",
       " ('god', 0.623152494430542),\n",
       " ('restaurant', 0.6084576845169067),\n",
       " ('macedonia', 0.6068291068077087)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('church')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('elizabeth', 0.7822296023368835),\n",
       " ('joshua', 0.7735604047775269),\n",
       " ('pamela', 0.7709989547729492),\n",
       " ('michelle', 0.7707457542419434),\n",
       " ('kyle', 0.7706141471862793),\n",
       " ('van', 0.7702932953834534),\n",
       " ('estrada', 0.7655531167984009),\n",
       " ('aaron', 0.7650339603424072),\n",
       " ('jeremy', 0.7648410201072693),\n",
       " ('nicholas', 0.7627168893814087)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('christian')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('muslim', 0.6500884890556335),\n",
       " ('latino', 0.6453544497489929),\n",
       " ('male', 0.6229506134986877),\n",
       " ('lived', 0.6065863370895386),\n",
       " ('violently', 0.5581741333007812),\n",
       " ('african-americans', 0.5568193197250366),\n",
       " ('gay', 0.5551722049713135),\n",
       " ('girls', 0.5549834966659546),\n",
       " ('impoverished', 0.554520308971405),\n",
       " ('pregnant', 0.5518898963928223)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('jewish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('restaurant', 0.708699643611908),\n",
       " ('church', 0.6867195963859558),\n",
       " ('congregation', 0.6682054996490479),\n",
       " ('backyard', 0.6602610945701599),\n",
       " ('mosque', 0.6552080512046814),\n",
       " ('motel', 0.6551803946495056),\n",
       " ('crowded', 0.631760835647583),\n",
       " ('nightclub', 0.6286150813102722),\n",
       " ('trailer', 0.6282986402511597),\n",
       " ('machete', 0.6265313625335693)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('synagogue')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking at names of presidents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('obama', 0.7955515384674072),\n",
       " ('barack', 0.6396726369857788),\n",
       " ('bush', 0.6285247802734375),\n",
       " ('biden', 0.6207939982414246),\n",
       " ('keeper', 0.5928440093994141),\n",
       " ('chun', 0.5360488295555115),\n",
       " ('president', 0.5348680019378662),\n",
       " ('hrc', 0.5300405621528625),\n",
       " ('committee', 0.5146782398223877),\n",
       " ('formally', 0.5133247375488281)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('trump')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('barack', 0.8377145528793335),\n",
       " ('trump', 0.795551598072052),\n",
       " ('president', 0.6371892690658569),\n",
       " ('keeper', 0.5776113867759705),\n",
       " ('committee', 0.54123455286026),\n",
       " ('championed', 0.5251625776290894),\n",
       " ('biden', 0.5005807876586914),\n",
       " ('declassification', 0.49753686785697937),\n",
       " ('clemency', 0.49681779742240906),\n",
       " ('interagency', 0.49499958753585815)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('obama')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('biden', 0.6811291575431824),\n",
       " ('clinton', 0.6657521724700928),\n",
       " ('trump', 0.6285247802734375),\n",
       " ('leahy', 0.622563898563385),\n",
       " ('wiley', 0.6110332012176514),\n",
       " ('cooke', 0.6040787100791931),\n",
       " ('roosevelt', 0.6008068323135376),\n",
       " ('ceo', 0.5986308455467224),\n",
       " ('hillary', 0.5924505591392517),\n",
       " ('janet', 0.5904005169868469)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('bush')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking at different races"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('horns', 0.7097222208976746),\n",
       " ('elephant', 0.6883867979049683),\n",
       " ('ivory', 0.6882531642913818),\n",
       " ('rhinoceros', 0.6848993301391602),\n",
       " ('coral', 0.6826488971710205),\n",
       " ('rhino', 0.6808892488479614),\n",
       " ('powder', 0.6472471356391907),\n",
       " ('deer', 0.6288437843322754),\n",
       " ('synthetic', 0.6279706954956055),\n",
       " ('turtle', 0.6259269118309021)]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('native', 0.7752351760864258),\n",
       " ('samoa', 0.7111507058143616),\n",
       " ('indians', 0.7051369547843933),\n",
       " ('natives', 0.6471330523490906),\n",
       " ('latino', 0.6406341791152954),\n",
       " ('muslim', 0.6262638568878174),\n",
       " ('eels', 0.6235378980636597),\n",
       " ('dream', 0.6160451769828796),\n",
       " ('african-american', 0.5895832777023315),\n",
       " ('asian', 0.5815019011497498)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('african')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('hispanic', 0.7146862149238586),\n",
       " ('interracial', 0.7027782797813416),\n",
       " ('female', 0.674068808555603),\n",
       " ('woman', 0.6406142711639404),\n",
       " ('predominantly', 0.6387567520141602),\n",
       " ('arrestee', 0.6347012519836426),\n",
       " ('male', 0.6327521204948425),\n",
       " ('latino', 0.6239545345306396),\n",
       " ('tenants', 0.6197518706321716),\n",
       " ('pregnant', 0.605392575263977)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('african-american')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('asian/pacific', 0.7177717685699463),\n",
       " ('african-american', 0.7146862745285034),\n",
       " ('borrowers', 0.6560112833976746),\n",
       " ('tenants', 0.6470454931259155),\n",
       " ('hispanics', 0.6349380016326904),\n",
       " ('female', 0.6300790905952454),\n",
       " ('islander', 0.6165289878845215),\n",
       " ('elderly', 0.5847363471984863),\n",
       " ('latino', 0.5710458755493164),\n",
       " ('pregnant', 0.558635950088501)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('hispanic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('amish', 0.6721432209014893),\n",
       " ('somali', 0.6702308058738708),\n",
       " ('predominantly', 0.6497206687927246),\n",
       " ('pregnant', 0.6465837955474854),\n",
       " ('bisexual', 0.6457959413528442),\n",
       " ('jewish', 0.6453544497489929),\n",
       " ('muslim', 0.6450033187866211),\n",
       " ('african', 0.6406341195106506),\n",
       " ('gay', 0.6405932307243347),\n",
       " ('african-american', 0.6239545345306396)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('latino')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('colombian', 0.7612261772155762),\n",
       " ('philippine', 0.665427565574646),\n",
       " ('salvadoran', 0.6654013395309448),\n",
       " ('colombia', 0.6227753162384033),\n",
       " ('romanian', 0.6035144329071045),\n",
       " ('canadian', 0.5905613899230957),\n",
       " ('brazilian', 0.5902118682861328),\n",
       " ('japanese', 0.5792630910873413),\n",
       " ('ukrainian', 0.5683580636978149),\n",
       " ('sinaloa', 0.5681815147399902)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('mexican')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('africa', 0.6763811111450195),\n",
       " ('east', 0.6450565457344055),\n",
       " ('descent', 0.6407113075256348),\n",
       " ('adjacent', 0.6246041655540466),\n",
       " ('sargasso', 0.6220220923423767),\n",
       " ('coral', 0.5958296060562134),\n",
       " ('fork', 0.5894865989685059),\n",
       " ('slope', 0.5855900049209595),\n",
       " ('african', 0.581501841545105),\n",
       " ('antiques', 0.5763877630233765)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('asian')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('emirates', 0.937386155128479),\n",
       " ('kingdom-based', 0.8184065818786621),\n",
       " ('kingdom', 0.8140121698379517),\n",
       " ('states-based', 0.786821186542511),\n",
       " ('states.the', 0.7718714475631714),\n",
       " ('therapeutics', 0.7572633624076843),\n",
       " ('airlines', 0.7431352138519287),\n",
       " ('ama', 0.7358680367469788),\n",
       " ('newark/columbia', 0.7345608472824097),\n",
       " ('delta', 0.7266659736633301)]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('arab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('russian', 0.695533812046051),\n",
       " ('vietnamese', 0.6044529676437378),\n",
       " ('chinese', 0.5887904763221741),\n",
       " ('syrian', 0.5766353607177734),\n",
       " ('venezuelan', 0.5460034608840942),\n",
       " ('foreign', 0.5426225662231445),\n",
       " ('prc', 0.5318738222122192),\n",
       " ('cuba', 0.5309222936630249),\n",
       " ('haitian', 0.527751624584198),\n",
       " ('clandestine', 0.5158886909484863)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('cuban')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('linden', 0.5440186262130737),\n",
       " ('vance', 0.5274033546447754),\n",
       " ('subcommittee', 0.5052212476730347),\n",
       " ('male', 0.4933227300643921),\n",
       " ('predominantly', 0.49181997776031494),\n",
       " ('house', 0.48627427220344543),\n",
       " ('supremacy/separatism', 0.4849385917186737),\n",
       " ('supremacist', 0.48281997442245483),\n",
       " ('african-american', 0.4760252833366394),\n",
       " ('40-year-old', 0.4663178324699402)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('white')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking at political parties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('mcclain', 0.4784807860851288),\n",
       " ('hoc', 0.4412802755832672),\n",
       " ('azerbaijan', 0.4404869079589844),\n",
       " ('ibori', 0.43897145986557007),\n",
       " ('soviet', 0.43720245361328125),\n",
       " ('panamanian', 0.4346635639667511),\n",
       " ('186,600', 0.43110916018486023),\n",
       " ('al-assad', 0.43054527044296265),\n",
       " ('governor', 0.42465198040008545),\n",
       " ('guinean', 0.4147576093673706)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('republican')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('electoral', 0.5676555037498474),\n",
       " ('political', 0.548275351524353),\n",
       " ('guinean', 0.5172073841094971),\n",
       " ('sovereignty', 0.5091806054115295),\n",
       " ('furthering', 0.48730793595314026),\n",
       " ('competitiveness', 0.4866899251937866),\n",
       " ('government-to-government', 0.4819740653038025),\n",
       " ('soccer', 0.4811435341835022),\n",
       " ('bidding', 0.4773799777030945),\n",
       " ('weakens', 0.4754981994628906)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('democratic') #democrat does not exist in corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('couch', 0.7727024555206299),\n",
       " ('shave', 0.738537609577179),\n",
       " ('denomination', 0.7216039299964905),\n",
       " ('noka', 0.7139601111412048),\n",
       " ('bcfl', 0.7085400819778442),\n",
       " ('dishwasher', 0.7064012885093689),\n",
       " ('retainers', 0.7049406170845032),\n",
       " ('irrelevant', 0.7001870274543762),\n",
       " ('exhibition', 0.6992817521095276),\n",
       " ('provocation', 0.6988611221313477)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('partisan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/madisonobrienjones/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('bidder', 0.6397594213485718),\n",
       " ('consultant', 0.6176154613494873),\n",
       " ('buyer', 0.5998591184616089),\n",
       " ('vendor', 0.5894853472709656),\n",
       " ('winning', 0.5756090879440308),\n",
       " ('person', 0.5597124099731445),\n",
       " ('parties', 0.5400657653808594),\n",
       " ('servicemember', 0.5336257219314575),\n",
       " ('entity', 0.5169912576675415),\n",
       " ('seller', 0.5057758092880249)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doj_vecs.most_similar('party')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}