Harvesting_Rules.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pickle\n",
    "import pandas as pd\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6.11 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3914"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                     S                                                                         \n",
      "                         ____________________________|_______________________________________________________________________   \n",
      "                        |                                               VP                                                   | \n",
      "                        |                        _______________________|___                                                 |  \n",
      "                      NP-SBJ                    |                           VP                                               | \n",
      "         _______________|___________________    |     ______________________|______________________________________          |  \n",
      "        |          |              ADJP      |   |    |        |                PP-CLR                              |         | \n",
      "        |          |           ____|____    |   |    |        |          ________|_________                        |         |  \n",
      "        NP         |          NP        |   |   |    |        NP        |                  NP                    NP-TMP      | \n",
      "   _____|____      |     _____|____     |   |   |    |     ___|____     |    ______________|__________        _____|_____    |  \n",
      " NNP        NNP    ,    CD        NNS   JJ  ,   MD   VB   DT       NN   IN  DT             JJ         NN    NNP          CD  . \n",
      "  |          |     |    |          |    |   |   |    |    |        |    |   |              |          |      |           |   |  \n",
      "Pierre     Vinken  ,    61       years old  ,  will join the     board  as  a         nonexecutive director Nov.         29  . \n",
      "\n"
     ]
    }
   ],
   "source": [
    "pTrees = nltk.corpus.treebank.parsed_sents()\n",
    "%time len(pTrees) #hm... so counting through all of the trees takes about 5 seconds, but this isn't even touching any part of the tree.\n",
    "pTrees[0].pretty_print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pTreeProductions = []\n",
    "for tree in pTrees:\n",
    "    pTreeProductions.append(tree.productions())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3914"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[S -> NP-SBJ VP .,\n",
       " NP-SBJ -> NP , ADJP ,,\n",
       " NP -> NNP NNP,\n",
       " NNP -> 'Pierre',\n",
       " NNP -> 'Vinken',\n",
       " , -> ',',\n",
       " ADJP -> NP JJ,\n",
       " NP -> CD NNS,\n",
       " CD -> '61',\n",
       " NNS -> 'years',\n",
       " JJ -> 'old',\n",
       " , -> ',',\n",
       " VP -> MD VP,\n",
       " MD -> 'will',\n",
       " VP -> VB NP PP-CLR NP-TMP,\n",
       " VB -> 'join',\n",
       " NP -> DT NN,\n",
       " DT -> 'the',\n",
       " NN -> 'board',\n",
       " PP-CLR -> IN NP,\n",
       " IN -> 'as',\n",
       " NP -> DT JJ NN,\n",
       " DT -> 'a',\n",
       " JJ -> 'nonexecutive',\n",
       " NN -> 'director',\n",
       " NP-TMP -> NNP CD,\n",
       " NNP -> 'Nov.',\n",
       " CD -> '29',\n",
       " . -> '.']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pTreeProductions)\n",
    "pTreeProductions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getAllInstances(wordType):\n",
    "    instances = []\n",
    "    for tree in pTreeProductions:\n",
    "        for rule in tree:\n",
    "            text = str(rule)\n",
    "            left, right = text.split(\" -> \")\n",
    "            #print(\"Left, Rule:\")\n",
    "            #print(left)\n",
    "            #print(rule)\n",
    "            #print()\n",
    "            if(left == wordType):\n",
    "                instances.append(rule)\n",
    "    return instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5834"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adjs = getAllInstances('JJ')\n",
    "len(adjs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open('cats.pkl','rb') \n",
    "cats = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NN\n"
     ]
    }
   ],
   "source": [
    "for cat in cats:\n",
    "    if(cat == 'NN'):\n",
    "        print(cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cats = list(cats)\n",
    "type(cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading all instances of:  ,\n",
      "Wall time: 1.23 s\n",
      "Loading all instances of:  .\n",
      "Wall time: 1.22 s\n",
      "Loading all instances of:  ADJP\n",
      "Wall time: 1.3 s\n",
      "Loading all instances of:  ADVP\n",
      "Wall time: 1.34 s\n",
      "Loading all instances of:  CC\n",
      "Wall time: 1.29 s\n",
      "Loading all instances of:  CD\n",
      "Wall time: 1.26 s\n",
      "Loading all instances of:  DT\n",
      "Wall time: 1.33 s\n",
      "Loading all instances of:  EX\n",
      "Wall time: 1.21 s\n",
      "Loading all instances of:  FRAG\n",
      "Wall time: 1.24 s\n",
      "Loading all instances of:  IN\n",
      "Wall time: 1.28 s\n",
      "Loading all instances of:  JJ\n",
      "Wall time: 1.69 s\n",
      "Loading all instances of:  JJR\n",
      "Wall time: 1.26 s\n",
      "Loading all instances of:  JJS\n",
      "Wall time: 1.23 s\n",
      "Loading all instances of:  MD\n",
      "Wall time: 1.24 s\n",
      "Loading all instances of:  NN\n",
      "Wall time: 1.3 s\n",
      "Loading all instances of:  NNP\n",
      "Wall time: 1.26 s\n",
      "Loading all instances of:  NNPS\n",
      "Wall time: 1.23 s\n",
      "Loading all instances of:  NNS\n",
      "Wall time: 1.26 s\n",
      "Loading all instances of:  NP\n",
      "Wall time: 1.29 s\n",
      "Loading all instances of:  NP-TMP\n",
      "Wall time: 1.32 s\n",
      "Loading all instances of:  PDT\n",
      "Wall time: 1.23 s\n",
      "Loading all instances of:  PP\n",
      "Wall time: 1.23 s\n",
      "Loading all instances of:  PRP\n",
      "Wall time: 1.55 s\n",
      "Loading all instances of:  PRP$\n",
      "Wall time: 1.35 s\n",
      "Loading all instances of:  PRT\n",
      "Wall time: 1.25 s\n",
      "Loading all instances of:  QP\n",
      "Wall time: 1.24 s\n",
      "Loading all instances of:  RB\n",
      "Wall time: 1.25 s\n",
      "Loading all instances of:  RBR\n",
      "Wall time: 1.22 s\n",
      "Loading all instances of:  RBS\n",
      "Wall time: 1.21 s\n",
      "Loading all instances of:  ROOT\n",
      "Wall time: 1.29 s\n",
      "Loading all instances of:  RP\n",
      "Wall time: 1.26 s\n",
      "Loading all instances of:  S\n",
      "Wall time: 1.21 s\n",
      "Loading all instances of:  SBAR\n",
      "Wall time: 1.27 s\n",
      "Loading all instances of:  SINV\n",
      "Wall time: 1.42 s\n",
      "Loading all instances of:  TO\n",
      "Wall time: 1.57 s\n",
      "Loading all instances of:  UCP\n",
      "Wall time: 1.43 s\n",
      "Loading all instances of:  VB\n",
      "Wall time: 1.23 s\n",
      "Loading all instances of:  VBD\n",
      "Wall time: 1.22 s\n",
      "Loading all instances of:  VBG\n",
      "Wall time: 1.28 s\n",
      "Loading all instances of:  VBN\n",
      "Wall time: 1.28 s\n",
      "Loading all instances of:  VBP\n",
      "Wall time: 1.24 s\n",
      "Loading all instances of:  VBZ\n",
      "Wall time: 1.24 s\n",
      "Loading all instances of:  VP\n",
      "Wall time: 1.29 s\n",
      "Loading all instances of:  WDT\n",
      "Wall time: 1.22 s\n",
      "Loading all instances of:  WHADVP\n",
      "Wall time: 1.24 s\n",
      "Loading all instances of:  WHNP\n",
      "Wall time: 1.25 s\n",
      "Loading all instances of:  WP\n",
      "Wall time: 1.21 s\n",
      "Loading all instances of:  WRB\n",
      "Wall time: 1.53 s\n"
     ]
    }
   ],
   "source": [
    "cat_lists = [] #list of lists\n",
    "for cat in sorted(cats):\n",
    "    print(\"Loading all instances of: \", cat)\n",
    "    %time cat_lists.append(getAllInstances(cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CD -> '61'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_lists[5][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getRuleProb(wordType, key, isLeaf):\n",
    "    tagIndex = 0\n",
    "    for tag in cat_lists:\n",
    "        try: #needs this try/except because there's one tag that's just empty, the line below errors out.\n",
    "            raw = str(cat_lists[tagIndex][0])\n",
    "        except:\n",
    "            continue\n",
    "        if(wordType == raw.split(\" -> \")[0]): #now at this point, we're on the list of entries for the tag. count hits/total\n",
    "            hits = 0 #^ this must be true for 'hits' to initialize, if it isn't then the tag is not in the sample\n",
    "            total = 0\n",
    "            for rule in cat_lists[tagIndex]:\n",
    "                #print(rule)\n",
    "                text = str(rule)\n",
    "                left, right = text.split(\" -> \")\n",
    "                #print(left, right)\n",
    "                if(isLeaf):\n",
    "                    if(key == right[1:-1] or key == right[1:-1].lower()):\n",
    "                        hits += 1\n",
    "                else:\n",
    "                        if(key == right):\n",
    "                            hits += 1\n",
    "                total += 1\n",
    "        tagIndex += 1\n",
    "    try:\n",
    "        print(\"Hits: \", hits, \"\\nTotal: \", total)\n",
    "    except:\n",
    "        print(\"This tag was not found in the sample of the Penn Treebank.\") #FW, LS, POS, UH, RP, SYM\n",
    "        return\n",
    "    return hits/total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hits:  0 \n",
      "Total:  5834\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getRuleProb('JJ', 'warm', True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hits:  740 \n",
      "Total:  23724\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.031192041814196596"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getRuleProb('NP', 'DT JJ NN', False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Getting Surprisal***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import StanfordPOSTagger\n",
    "from nltk.parse import stanford\n",
    "from nltk.parse import CoreNLPParser\n",
    "parser = CoreNLPParser(url='http://localhost:9000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "foo = parser.raw_parse(\"The cow jumped over the moon.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   ROOT                     \n",
      "                    |                        \n",
      "                    S                       \n",
      "      ______________|_____________________   \n",
      "     |                   VP               | \n",
      "     |         __________|___             |  \n",
      "     |        |              PP           | \n",
      "     |        |      ________|___         |  \n",
      "     NP       |     |            NP       | \n",
      "  ___|___     |     |         ___|___     |  \n",
      " DT      NN  VBD    IN       DT      NN   . \n",
      " |       |    |     |        |       |    |  \n",
      "The     cow jumped over     the     moon  . \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for tree in foo:\n",
    "    foo = tree\n",
    "    tree.pretty_print()\n",
    "    rules = tree.productions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This tag was not found in the sample of the Penn Treebank.\n",
      "This tag was not found in the sample of the Penn Treebank.\n"
     ]
    }
   ],
   "source": [
    "getRuleProb(\"ROOT\", \"S\", False)\n",
    "getRuleProb(\"S\", \"NP VP .\", False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "8650"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "7857"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['S -> NP-SBJ VP .',\n",
       " 'S -> NP-SBJ VP .',\n",
       " 'S -> NP-SBJ-1 VP .',\n",
       " 'S -> NP-SBJ NP-PRD',\n",
       " 'S -> S-TPC-1 , NP-SBJ VP .',\n",
       " 'S -> S-TPC-2 , NP-SBJ VP .',\n",
       " 'S -> NP-SBJ VP',\n",
       " 'S -> NP-SBJ VP',\n",
       " 'S -> NP-SBJ VP .',\n",
       " 'S -> NP-SBJ VP']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_lists[29][:10]\n",
    "len(cat_lists[31])\n",
    "sRules = []\n",
    "for rule in cat_lists[31]:\n",
    "    text = str(rule)\n",
    "    if 'NP' in text:\n",
    "        sRules.append(text)\n",
    "len(sRules)\n",
    "sRules[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[ROOT -> S, S -> NP VP ., NP -> DT NN, DT -> 'The', NN -> 'cow']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This tag was not found in the sample of the Penn Treebank.\n",
      "Hits:  0 \n",
      "Total:  23724\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for +: 'NoneType' and 'float'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-52-5769bbc89307>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrules\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mrules\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mpoint\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetRuleProb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"S\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"NP VP .\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mgetRuleProb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"NP\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"DT NN DT\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mgetRuleProb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"DT\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"The\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mgiven\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpoint\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mgetRuleProb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"NN\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"cow\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'NoneType' and 'float'"
     ]
    }
   ],
   "source": [
    "len(rules)\n",
    "rules[:5]\n",
    "given = getRuleProb(\"S\", \"NP VP .\", False) * getRuleProb(\"NP\", \"DT NN DT\", False) * getRuleProb(\"DT\", \"The\", True)\n",
    "point = given + getRuleProb(\"NN\", \"cow\", True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hits:  4753 \n",
      "Total:  8165\n",
      "Hits:  75 \n",
      "Total:  9857\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "76.50593345580731"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "point = getRuleProb(\"DT\", \"the\", True)\n",
    "given = getRuleProb(\"IN\", \"over\", True)\n",
    "surprisal = point/given\n",
    "surprisal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}