hist3368-week12-distinctiveness-for-teaching-purposes.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hist 3368 - Measuring Distinctiveness with Log Likelihood "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# For teaching purposes\n",
    "\n",
    "#### By Jo Guldi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Demonstrating word vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "mystring =  [\"Peter Piper picked a peck of pickled peppers. A peck of pickled peppers Peter Piper picked. If Peter Piper picked a peck of pickled peppers, Where's the peck of pickled peppers Peter Piper picked?\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['if', 'of', 'peck', 'peppers', 'peter', 'picked', 'pickled', 'piper', 'the', 'where']\n",
      "[[1 4 4 4 4 4 4 4 1 1]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np, pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(stop_words = None)\n",
    "\n",
    "vectors = vectorizer.fit_transform(mystring)\n",
    "print(vectorizer.get_feature_names())\n",
    "print(vectors.toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary of Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Software"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import some software\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk import word_tokenize          \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, articles):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(articles)]\n",
    "import adjustText\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load some Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cd /scratch/group/history/hist_3368-jguldi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "congress = pd.read_csv(\"congress1967-2010.csv\")\n",
    "#congress = pd.read_csv(\"eighties_data.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean up congress\n",
    "congress = congress[congress['year'] == 1968]\n",
    "clean_congress = congress.copy()\n",
    "clean_congress['speech'] = clean_congress['speech'].str.replace('[^\\w\\s]','') # remove punctuation\n",
    "clean_congress['speech'] = clean_congress['speech'].str.replace('\\d+', '') # for digits\n",
    "clean_congress['speech'] = clean_congress['speech'].str.replace(r'(\\b\\w{1}\\b)', '') # for short words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "congress[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_congress[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Format Data Around Units of Interest With One String per Unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# format the data around our research into speakers\n",
    "top_speakers = clean_congress.groupby('speaker').agg({'speech': ' '.join, 'word_count': 'sum'}).sort_values('word_count', ascending = False)[:10]\n",
    "top_speakers = top_speakers[top_speakers.index != 'The PRESIDING OFFICER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_speakers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make Word Vectors -- One for Each Unit of Interest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a word vector and get some information from it\n",
    "vectorizer = CountVectorizer(\n",
    "    max_features=10000, \n",
    "    lowercase=True, \n",
    "    stop_words = 'english',\n",
    "    ngram_range=(1, 2), \n",
    "    analyzer = \"word\",\n",
    "    #tokenizer=LemmaTokenizer()\n",
    "    )\n",
    "\n",
    "vectorized = vectorizer.fit_transform(top_speakers['speech'])\n",
    "vectorized"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make the Word Vectors Readable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'np' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-6dde77f098a9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mall_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvectorizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_feature_names\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mspeaker_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtop_speakers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m vectors_dataframe = pd.DataFrame(vectorized.todense(), # the matrix we saw above is turned into a dataframe\n\u001b[1;32m      5\u001b[0m                                  \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_words\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"
     ]
    }
   ],
   "source": [
    "all_words = np.array(vectorizer.get_feature_names())\n",
    "speaker_names = list(top_speakers.axes[0].to_numpy())\n",
    "\n",
    "vectors_dataframe = pd.DataFrame(vectorized.todense(), # the matrix we saw above is turned into a dataframe\n",
    "                                 columns=all_words,\n",
    "                                 index = speaker_names\n",
    "                                 )\n",
    "vectors_dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute some baseline numbers about the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "speaker_words_total = vectors_dataframe.sum(axis=1)\n",
    "word_totals = vectors_dataframe.sum(axis=0) \n",
    "total_corpus_words = sum(word_totals)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Measure distinctiveness with log likelihood"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## create an empty dataframe\n",
    "speakers_loglikelihood = []\n",
    "\n",
    "## loop through every speaker in speaker_names\n",
    "for speaker_id, speaker in enumerate(speaker_names):\n",
    "    loglikely = []\n",
    "    # loop through every word in the wordcount vector:\n",
    "    for word_id in vectorized[speaker_id].indices:\n",
    "        \n",
    "        a = vectors_dataframe.iat[speaker_id, word_id] #  word in speaker\n",
    "        b = word_totals[word_id] - a  # # word in remaining speakers\n",
    "        c = speaker_words_total[speaker_id] - a #  not word in speaker\n",
    "        d = total_corpus_words - a - b - c # not word in remaining speakers\n",
    "       \n",
    "        E1 = (a + c) * (a + b) / total_corpus_words  \n",
    "        E2 = (b + d) * (a + b) / total_corpus_words \n",
    "        \n",
    "        LL = 2 * (a * np.log(a / E1)) # the log likelihood equation\n",
    "        if (b > 0):\n",
    "            LL += 2 * b * np.log(b / E2)\n",
    "        \n",
    "        loglikely.append((LL, all_words[word_id])) # add the log likelihood score to the end of a new dataframe\n",
    "\n",
    "    loglikely = sorted(loglikely, reverse=True) # the loop hits this every time it cycles through all the words in one speaker. \n",
    "    speakers_loglikelihood.append(loglikely) # add on another speaker\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"The 20 most distinctive words of each speaker. The words are listed from high to low ranking\")\n",
    "print(\"-------------------------------------------\\n\")\n",
    "for i, speaker in enumerate(speaker_names):\n",
    "    print(speaker + \": \", end='')\n",
    "    distinct_words = [word[1] for word in speakers_loglikelihood[i][:20]]\n",
    "    print(distinct_words)\n",
    "    print(\"\\n-----------------------------\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizing the distinctiveness of the language of each speaker"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that we're going to make a big plot.\n",
    "\n",
    "***This might take a minute or two to run.***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cd ~/digital-history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from adjustText import adjust_text\n",
    "\n",
    "# change the figure's size here\n",
    "plt.figure(figsize=(10,10), dpi = 500)\n",
    "\n",
    "# style\n",
    "plt.style.use('seaborn-darkgrid') # this gives us a grid with a dark background.  you can play with this to change the style.\n",
    "  \n",
    "# create a color palette\n",
    "palette = plt.get_cmap('hsv') # this tells matplotlib what colors to use.  you can play with this to change the colors.\n",
    "\n",
    "# start a counter at 0\n",
    "num = 0\n",
    "\n",
    "# create an empty list\n",
    "texts = []\n",
    "\n",
    "# this is the for loop that creates multiple plots.  \n",
    "for i, speaker in enumerate(speaker_names):\n",
    "        num += 14 # num tells the plot to choose a different color for each speaker\n",
    "        distinct_words = speakers_loglikelihood[i][:20] # plot the top twenty words by LL-score\n",
    "        for word in distinct_words: # for each word-per-speaker instance, plot the ll_score on the y axis\n",
    "            ll_score = word[0] # find just the ll-score from speakers_loglikelihood\n",
    "            word_label = word[1] # find just the keyword name from speakers_loglikelihood\n",
    "            plt.scatter( # draw a dot for each word\n",
    "                    speaker, # with speaker on the x axis\n",
    "                    ll_score, # and ll_score on the y axis\n",
    "                    color=palette(num), # using a different color for each speaker\n",
    "                    linewidth=1, \n",
    "                    edgecolors = 'b',\n",
    "                    s = 55, # size\n",
    "                    alpha=0.3, # make the dots slightly transparent\n",
    "                    label=speaker) # label each dot with the name of the word\n",
    "            texts.append(plt.text(speaker, ll_score, word_label)) # save these coordinates to be used in labeling\n",
    "\n",
    "# Add legend\n",
    "plt.xticks(rotation=90)\n",
    " \n",
    "# Add titles\n",
    "plt.title(\"Figure 1: Highest Log-Likelihood Scored Words per Speaker\", fontsize=30, fontweight=0, color='Red')\n",
    "plt.xlabel(\"Speaker\")\n",
    "plt.ylabel(\"Distinctiveness of Words, Measured by LL Score\")\n",
    "\n",
    "# Code to help with overlapping labels -- may take a minute to run\n",
    "adjust_text(texts, force_points=0.2, force_text=0.2,\n",
    "            expand_points=(1, 1), expand_text=(1, 1),\n",
    "            arrowprops=dict(arrowstyle=\"-\", color='black', lw=0.5))\n",
    "\n",
    "# save it\n",
    "fig1 = plt.gcf()\n",
    "plt.show()\n",
    "plt.draw()\n",
    "fig1.savefig('LL-fig1.jpg', dpi=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A 2-D comparison of two speakers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create a dataframe from just two speakers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ***The x coordinate will be how distinctive each word is for Javits; the y coordinate will be how distinctive each word is for Long.  Change the speakers in question by changingn in speakers_loglikelihood[n].***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a new dataframe of the scores and words from both Javits and Long\n",
    "xcoords =  pd.DataFrame(columns=list(['word_label', 'x_llscore']))\n",
    "ycoords = pd.DataFrame(columns=list(['word_label', 'y_llscore']))\n",
    "\n",
    "# get all the words from JAVITS\n",
    "distinctwords = speakers_loglikelihood[0]\n",
    "for word in distinctwords: # for each word-per-cspeaker instance, plot the ll_score on the y axis\n",
    "    x_llscore = word[0] # find just the ll-score from speakers_loglikelihood\n",
    "    word_label = word[1] # find just the keyword name from speakers_loglikelihood\n",
    "    speaker = 'JAVITS'\n",
    "    xcoords.loc[-1] = [word_label, x_llscore]\n",
    "    xcoords.index = xcoords.index + 1  # shifting index\n",
    "    xcoords = xcoords.sort_index()  # sorting by index\n",
    "\n",
    "# get all the words from LONG\n",
    "distinctwords2 = speakers_loglikelihood[1] \n",
    "for word in distinctwords2: # for each word-per-speaker instance, plot the ll_score on the y axis\n",
    "    y_llscore = word[0] # find just the ll-score from speakers_loglikelihood\n",
    "    word_label = word[1] # find just the keyword name from speakers_loglikelihood\n",
    "    speaker = 'LONG'\n",
    "    ycoords.loc[-1] = [word_label, y_llscore]\n",
    "    ycoords.index = ycoords.index + 1  # shifting index\n",
    "    ycoords = ycoords.sort_index()  # sorting by index\n",
    "\n",
    "coords = pd.merge(xcoords, ycoords, on='word_label')\n",
    "coords = coords.dropna(axis = 0, how ='any') # drop rows with any NA's"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install adjustText --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from adjustText import adjust_text\n",
    "\n",
    "# change the figure's size here\n",
    "plt.figure(figsize=(10,10), dpi = 500)\n",
    "\n",
    "# label each dot with the name of the word -- note that we have to use a \"for\" loop for this to work; plt.annotate only plots\n",
    "# one label per iteration!\n",
    "for i, txt in enumerate(coords['word_label']):\n",
    "    # draw a dot for each word\n",
    "    plt.scatter( \n",
    "        coords['x_llscore'][i], #x axis\n",
    "        coords['y_llscore'][i], # y axis\n",
    "        linewidth=1, \n",
    "        s = 55, # dot size\n",
    "        alpha=0.2)  # dot transparency\n",
    "    # make a label for each word\n",
    "    plt.annotate(\n",
    "        txt, \n",
    "        (coords['x_llscore'][i], # one x llscore at a time\n",
    "         coords['y_llscore'][i]), # one y llscore at a time\n",
    "        alpha=0.3 # i've made the fonts transparent as well.  you could play with color and size if you wanted to. \n",
    "    )\n",
    "\n",
    "plt.xticks(rotation=90)\n",
    "\n",
    "# logarithmic axes make big things big and small things small\n",
    "plt.xscale('log')\n",
    "plt.yscale('log')  \n",
    "\n",
    "# Add titles\n",
    "plt.title(\"Figure 2: Highest Log-Likelihood Scored Words per Speaker\", fontsize=30, fontweight=0, color='Red')\n",
    "plt.xlabel(\"How Distinctive Each Word is of Mr. Javits\")\n",
    "plt.ylabel(\"How Distinctive Each Word is of Mr. Long\")\n",
    "\n",
    "\n",
    "# save it\n",
    "fig1 = plt.gcf()\n",
    "plt.show()\n",
    "plt.draw()\n",
    "fig1.savefig('LL-fig2.jpg', dpi=500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From Data to Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cd /scratch/group/history/hist_3368-jguldi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "congress = pd.read_csv(\"congress1967-2010.csv\")\n",
    "congress = congress[congress['year'] == 1968]\n",
    "top_speakers_speeches = congress[congress['speaker'].isin(top_speakers.index)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup: Make a KWIC Dictionary for a Single Speaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string    \n",
    "long_speeches = top_speakers_speeches[top_speakers_speeches['speaker'] == \"Mr. LONG of Louisiana\"]['speech']\n",
    "long_speeches = ' '.join(long_speeches).lower() # join back together and lowercase\n",
    "long_speeches = long_speeches.translate(str.maketrans('', '', string.punctuation)) # remove punctuation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add some functions for defining ngrams "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getNGrams(wordlist, n):\n",
    "    ngrams = []\n",
    "    for i in range(len(wordlist)-(n-1)):\n",
    "        ngrams.append(wordlist[i:i+n])\n",
    "    return ngrams\n",
    "\n",
    "def nGramsToKWICDict(ngrams):\n",
    "    keyindex = len(ngrams[0]) // 2\n",
    "\n",
    "    kwicdict = {}\n",
    "\n",
    "    for k in ngrams:\n",
    "        if k[keyindex] not in kwicdict:\n",
    "            kwicdict[k[keyindex]] = [k]\n",
    "        else:\n",
    "            kwicdict[k[keyindex]].append(k)\n",
    "    return kwicdict\n",
    "\n",
    "\n",
    "def prettyPrintKWIC(kwic):\n",
    "    n = len(kwic)\n",
    "    keyindex = n // 2\n",
    "    width = 20\n",
    "\n",
    "    outstring = ' '.join(kwic[:keyindex]).rjust(width*keyindex)\n",
    "    outstring += str(kwic[keyindex]).center(len(kwic[keyindex])+6)\n",
    "    outstring += ' '.join(kwic[(keyindex+1):])\n",
    "\n",
    "    return outstring\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fullwordlist = long_speeches.split()\n",
    "ngrams = getNGrams(fullwordlist, 20)\n",
    "worddict = nGramsToKWICDict(ngrams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Doing Research With Log Likelihood and KWIC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "speakers_loglikelihood[1][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# output KWIC for target word\n",
    "target = 'gun'\n",
    "outstr = '<pre>'\n",
    "if target in worddict:\n",
    "    for k in worddict[target]:\n",
    "        outstr += prettyPrintKWIC(k)\n",
    "        outstr += '             '\n",
    "else:\n",
    "    outstr += 'Keyword not found in source'\n",
    "\n",
    "outstr += '</pre>'\n",
    "outstr"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10 (main, Feb 25 2022, 18:07:24) \n[Clang 13.0.0 (clang-1300.0.29.30)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}