mining_gutenberg_for_relations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gymnasion Data Processing\n",
    "\n",
    "Here I'm going to mine some chunk of Project Gutenberg texts for `(adj,noun)` and `(noun,verb,object)` relations using mostly SpaCy and textacy.  Extracting them is easy.  Filtering out the chaff is not so easy.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# -*- coding: utf-8 -*-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "from collections import defaultdict\n",
    "from nltk import ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from textacy import extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load in some randomly chosen Gutenberg texts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "gb_files = [f for f in os.listdir(\"/Users/kyle/Documents/gutenberg_samples/\") if f.startswith('gb_')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a function to extract `(adj,noun)` relations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'small', u'frog'), (u'only', u'one')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def extract_adj2nouns(tempspacy):\n",
    "    \"\"\"\n",
    "    For a sentence like \"I ate the small frog.\" returns [(small, frog)].\n",
    "    lemmatizes the noun, lowers the adjective\n",
    "    \"\"\"\n",
    "    nouns = [\"NN\",\"NNS\"]\n",
    "    adj_noun_tuples = []\n",
    "    for token in tempspacy:\n",
    "        if token.dep_==\"amod\":\n",
    "            if token.head.tag_ in nouns:\n",
    "                adj_noun_tuples.append((token.text.lower(),token.head.lemma_))\n",
    "    return adj_noun_tuples\n",
    "                                       \n",
    "extract_adj2nouns(nlp(u\"The small frogs were not the only ones there.  The dog walked itself through the house.\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Textacy extracts `(s,v,o)` triples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(husband, ignored, wife)\n"
     ]
    }
   ],
   "source": [
    "for triple in extract.subject_verb_object_triples(nlp(u\"The husband ignored his wife.\")):\n",
    "    print triple"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I want to loop through a bunch of Gutenberg texts that I've randomly downloaded with the Gutenberg python package.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from langdetect import detect ## to make sure texts are english"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from unidecode import unidecode ## to crudely deal with text encoding issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "noun2adj = defaultdict(list)\n",
    "noun2object = defaultdict(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [1:14:54<00:00,  6.57s/it] \n"
     ]
    }
   ],
   "source": [
    "noun2adj_tuples = []\n",
    "svo_triples = []\n",
    "errors = 0\n",
    "\n",
    "for fy in tqdm(gb_files[:1000]):\n",
    "    with open(\"/Users/kyle/Documents/gutenberg_samples/\"+fy,'r') as f:\n",
    "        tempdata = f.read()\n",
    "        try:\n",
    "            if detect(tempdata)==\"en\": ## check english\n",
    "                tempspacy = nlp(tempdata.decode('utf-8'))\n",
    "\n",
    "                ### adjectives\n",
    "                try:\n",
    "                    for pair in extract_adj2nouns(tempspacy):\n",
    "                        noun2adj_tuples.append(pair)\n",
    "                except:\n",
    "                    pass\n",
    "\n",
    "                ### svo triples\n",
    "                try:\n",
    "                    gutenberg_svo_triples = extract.subject_verb_object_triples(tempspacy)\n",
    "                    for trip in gutenberg_svo_triples:\n",
    "                        svo_triples.append(trip)\n",
    "                except:\n",
    "                    pass\n",
    "        except:\n",
    "            errors+=1\n",
    "            \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many pairs (not unique) do I have of `(adj,noun)` relations?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2285591"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(noun2adj_tuples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Of `(s,v,o)` relations?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2403881"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(svo_triples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspecting the data so far..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `(adj, noun)` relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'uncomplaining', u'spirit'),\n",
       " (u'white', u'drapery'),\n",
       " (u'second', u'address'),\n",
       " (u'little', u'hillock'),\n",
       " (u'little', u'benefit'),\n",
       " (u'voluntary', u'contribution'),\n",
       " (u'beautiful', u'bird'),\n",
       " (u'political', u'connection'),\n",
       " (u'speculative', u'look'),\n",
       " (u'whimsical', u'portrait'),\n",
       " (u'such', u'practice'),\n",
       " (u'delightful', u'time'),\n",
       " (u'wounded', u'man'),\n",
       " (u'clean', u'shore'),\n",
       " (u'single', u'course'),\n",
       " (u'worn', u'grass'),\n",
       " (u'hapless', u'prey'),\n",
       " (u'colored', u'shirt'),\n",
       " (u'leading', u'theory'),\n",
       " (u'useful', u'plant')]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(noun2adj_tuples,20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another way to inspect data: frequency distributions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk import FreqDist as fd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ADJ_noun_fd = fd([a for a,n in noun2adj_tuples])\n",
    "adj_NOUN_fd = fd([n for a,n in noun2adj_tuples])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'great', 48163),\n",
       " (u'other', 46884),\n",
       " (u'little', 46060),\n",
       " (u'old', 37059),\n",
       " (u'good', 34388),\n",
       " (u'own', 32720),\n",
       " (u'first', 29181),\n",
       " (u'many', 28795),\n",
       " (u'same', 27871),\n",
       " (u'such', 27821),\n",
       " (u'young', 24405),\n",
       " (u'new', 21912),\n",
       " (u'long', 19462),\n",
       " (u'few', 19209),\n",
       " (u'last', 18339),\n",
       " (u'small', 16753),\n",
       " (u'whole', 16372),\n",
       " (u'large', 14756),\n",
       " (u'more', 14084),\n",
       " (u'white', 12726),\n",
       " (u'several', 12081),\n",
       " (u'much', 11978),\n",
       " (u'next', 11899),\n",
       " (u'certain', 11299),\n",
       " (u'poor', 10796),\n",
       " (u'black', 10134),\n",
       " (u'high', 9956),\n",
       " (u'human', 8625),\n",
       " (u'full', 8498),\n",
       " (u'different', 8428),\n",
       " (u'only', 8320),\n",
       " (u'general', 8246),\n",
       " (u'best', 8235),\n",
       " (u'second', 7987),\n",
       " (u'big', 7245),\n",
       " (u'present', 6954),\n",
       " (u'public', 6900),\n",
       " (u'short', 6899),\n",
       " (u'various', 6855),\n",
       " (u'very', 6740)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ADJ_noun_fd.most_common(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'man', 50670),\n",
       " (u'time', 27787),\n",
       " (u'thing', 24171),\n",
       " (u'day', 21246),\n",
       " (u'woman', 13259),\n",
       " (u'way', 13226),\n",
       " (u'year', 12865),\n",
       " (u'eye', 12829),\n",
       " (u'hand', 12460),\n",
       " (u'part', 12177),\n",
       " (u'life', 11846),\n",
       " (u'people', 11307),\n",
       " (u'place', 11195),\n",
       " (u'girl', 10362),\n",
       " (u'one', 9507),\n",
       " (u'word', 9303),\n",
       " (u'friend', 8939),\n",
       " (u'face', 8731),\n",
       " (u'work', 8555),\n",
       " (u'house', 7599),\n",
       " (u'voice', 7344),\n",
       " (u'person', 7000),\n",
       " (u'night', 6969),\n",
       " (u'form', 6921),\n",
       " (u'lady', 6865),\n",
       " (u'child', 6706),\n",
       " (u'boy', 6630),\n",
       " (u'side', 6550),\n",
       " (u'line', 6486),\n",
       " (u'room', 6368),\n",
       " (u'fellow', 6307),\n",
       " (u'water', 5996),\n",
       " (u'power', 5948),\n",
       " (u'something', 5857),\n",
       " (u'manner', 5521),\n",
       " (u'world', 5453),\n",
       " (u'condition', 5368),\n",
       " (u'air', 5308),\n",
       " (u'light', 5302),\n",
       " (u'moment', 5275)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adj_NOUN_fd.most_common(40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ideas...\n",
    "\n",
    "So there are really two problems.  Looking at the frequency distribution tells me that some of the most common adjectives (e.g. \"few\", \"other\")) are undesirable, because they aren't closely tied to a noun.  That leaves are `green` is better to know than that leaves can be `other`.  (Also, certain common nouns are probably not as interesting, especially ones like `other`).  I have two intuitions: 1) really common relationships between adjectives and nouns are less interesting/desirable than less common ones, and 2) at the same time, I really don't want `(adj,noun)` pairs that are totally aberrant.  Regarding the second point, I could filter out any adjective that doesn't occur at least `n` in modification of a certain noun, but that really penalizes uncommon nouns (which won't have many adjectives modifying them).\n",
    "\n",
    "My plan:\n",
    "\n",
    "1. Filter out relations containing the most adjectives as well as a handful of annoying nouns\n",
    "2. Filter out those relations between words that are not strongly related according to a word2vec model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A handmade list of nouns to exclude."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ADJS_nouns_to_exclude = [word for word,count in ADJ_noun_fd.most_common(40)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'great', u'other', u'little', u'old', u'good', u'own', u'first', u'many', u'same', u'such', u'young', u'new', u'long', u'few', u'last', u'small', u'whole', u'large', u'more', u'white', u'several', u'much', u'next', u'certain', u'poor', u'black', u'high', u'human', u'full', u'different', u'only', u'general', u'best', u'second', u'big', u'present', u'public', u'short', u'various', u'very']\n"
     ]
    }
   ],
   "source": [
    "print ADJS_nouns_to_exclude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stops = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stops = stops + [\"whose\",\"less\",\"thee\",\"thine\",\"thy\",\"thou\",\"one\"] ##adjectives nouns\n",
    "stops = stops + [\"time\",\"thing\",\"one\",\"way\",\"part\",\"something\"] ##annoying nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "noun2adj = defaultdict(list)\n",
    "for a,n in noun2adj_tuples:\n",
    "    if n not in stops:\n",
    "        if (a not in ADJS_nouns_to_exclude) and (a not in stops):\n",
    "            noun2adj[n].append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "word2vec_path = \"/Users/kyle/Desktop/GoogleNews-vectors-negative300.bin\"\n",
    "model = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below I define a pretty confusing loop to go through the dictionary I just made to filter out those words that `(adj,noun)` pairs that are unrelated according to a word2vec model.  (Here I'm using just cosine similarity but I could probably maybe just measure the probability of the text according to the model.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 34122/34122 [00:47<00:00, 724.92it/s] \n"
     ]
    }
   ],
   "source": [
    "new_noun2adj = defaultdict(list)\n",
    "\n",
    "for k in tqdm(noun2adj.keys()):\n",
    "    adjs = []\n",
    "    for adj in noun2adj[k]:\n",
    "        try:\n",
    "            adjs.append((adj,model.similarity(adj,k)))\n",
    "        except:\n",
    "            pass\n",
    "    adjs.sort(key = lambda x: x[1], reverse=True)\n",
    "    adjs_ = []\n",
    "    for a,score in adjs:\n",
    "        if a not in adjs_:\n",
    "            adjs_.append(a)\n",
    "            \n",
    "    ## this is some weird hand-crafted logic to filter adjectives belonging to rare and common nouns differently...\n",
    "    ## the idea is to only take the cream of the crop when there are a lot of options --- i.e. when the noun is common\n",
    "    if len(adjs_)>20:\n",
    "        for adj in adjs_[:10]:\n",
    "            new_noun2adj[k].append(adj)\n",
    "    elif len(adjs_)>10:\n",
    "        for adj in adjs_[:10]:\n",
    "            new_noun2adj[k].append(adj)\n",
    "    elif len(adjs_)>2:\n",
    "        adj = adjs_[0]\n",
    "        new_noun2adj[k].append(adj)\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'uniform',\n",
       " u'tweed',\n",
       " u'worn',\n",
       " u'broadcloth',\n",
       " u'pink',\n",
       " u'cowboy',\n",
       " u'khaki',\n",
       " u'blue',\n",
       " u'floppy',\n",
       " u'plumed']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_noun2adj['hat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"data/noun2adj.json\",\"w\") as f:\n",
    "    json.dump(new_noun2adj,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  `(s,v,o)` triples..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svo_triples_reformatted = [(s.lemma_,v.lemma_,o.text) for s,v,o, in svo_triples]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspect data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'ruth', u'close', u'novel'),\n",
       " (u'alliance', u'be', u'together'),\n",
       " (u'give', u'be', u'procedure'),\n",
       " (u'jungle wall', u'be', u'line'),\n",
       " (u'-PRON-', u'bear', u'yoke'),\n",
       " (u'-PRON-', u'be', u'as'),\n",
       " (u'fred', u'take', u'habit'),\n",
       " (u'-PRON-', u'hunt', u'superintendent'),\n",
       " (u'officer', u'take', u'care'),\n",
       " (u'remedy', u'banish', u'tender Sentiments'),\n",
       " (u'-PRON-', u'begin', u'flinging'),\n",
       " (u'miss flagg', u'take', u'manuscript'),\n",
       " (u'-PRON-', u'may believe', u'charge'),\n",
       " (u'troop', u'offer', u'show'),\n",
       " (u'-PRON-', u'say', u'nothing'),\n",
       " (u'man', u'be call', u'mad'),\n",
       " (u'there', u'be', u'demand'),\n",
       " (u'miltiades', u'become', u'household problem'),\n",
       " (u'-PRON-', u'deceive', u'them'),\n",
       " (u'court', u'pronounce', u'defendant--')]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(svo_triples_reformatted,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Svo_fd = fd([s for s,v,o in svo_triples_reformatted])\n",
    "sVo_fd = fd([v for s,v,o in svo_triples_reformatted])\n",
    "svO_fd = fd([o for s,v,o, in svo_triples_reformatted])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'-PRON-', u'there', u'who', u'which', u'that', u'this', u'man', u'one', u'what', u'people', u'god', u'woman', u'father', u'all', u'boy', u'be', u'mother', u'some', u'thing', u'girl', u'king', u'other', u'thou', u'child', u'these', u'friend', u'name', u'\"', u'eye', u'word', u'nothing', u'person', u'have', u'lady', u'many', u'those', u'wife', u'son', u'face', u'life']\n"
     ]
    }
   ],
   "source": [
    "topS = [word for word,count in Svo_fd.most_common(40)]\n",
    "print topS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'be', u'have', u'give', u'take', u'seem', u'make', u'see', u'tell', u'know', u'have be', u'begin', u'say', u'find', u'want', u'call', u'leave', u'do', u'get', u'bring', u'hear', u'ask', u'become', u'show', u'send', u'come', u'keep', u'use', u'love', u'put', u'ought', u'hold', u'go', u'appear', u'wish', u'be not', u'would be', u'feel', u'reach', u'try', u'mean']\n"
     ]
    }
   ],
   "source": [
    "topV = [word for word,count in sVo_fd.most_common(40)]\n",
    "print topV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'him', u'it', u'me', u'you', u'them', u'her', u'to be', u'one', u'us', u'nothing', u'that', u'man', u'himself', u'more', u'something', u'way', u'place', u'time', u'thing', u'to', u'to do', u'all', u'this', u'anything', u'head', u'to see', u'to go', u'to make', u'to have', u'hand', u'themselves', u'to say', u'things', u'part', u'men', u'eyes', u'name', u'life', u'to take', u'herself']\n"
     ]
    }
   ],
   "source": [
    "topO = [word for word,count in svO_fd.most_common(40)]\n",
    "print topO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The loop below filters out an `(s,v,o)` triple if any one of its elements meets certain exclusionary conditions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svo_triples_filtered = []\n",
    "\n",
    "for s,v,o, in svo_triples_reformatted:\n",
    "    Sval,Vval,Oval=False,False,False\n",
    "    if len(s.split())==1: ## make sure it's not a complicated noun chunk\n",
    "        if s.lower() not in stops:  ## make sure it's not a stopword\n",
    "            Sval=True\n",
    "    if v not in topV:   ## make sure it's not really common\n",
    "        if len(v.split())==1:  ## make sure it's not a complicated verb chunk\n",
    "            if v.lower() not in stops:  ## make sure it's not a stopwords\n",
    "                Vval=True    \n",
    "    if len(o.split())==1:    ### make sure it's not a complicated noun chunk\n",
    "        if o.lower() not in stops:  ### make sure it's not a stopword\n",
    "            if o.lower()==o:  ## this is kind of a hack to exclude proper nouns\n",
    "                if o.endswith(\"ing\")==False:  ## filter out annoying present participles\n",
    "                    Oval=True\n",
    "    if (Sval,Vval,Oval)==(True,True,True):\n",
    "        svo_triples_filtered.append((s,v,o))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "noun2v_o = defaultdict(list)\n",
    "for s,v,o in svo_triples_filtered:\n",
    "    noun2v_o[s].append((v,o))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'decline', u'share'),\n",
       " (u'request', u'presence'),\n",
       " (u'present', u'offerings'),\n",
       " (u'beg', u'monks'),\n",
       " (u'present', u'offerings'),\n",
       " (u'receive', u'increase'),\n",
       " (u'divide', u'relics'),\n",
       " (u'employ', u'men'),\n",
       " (u'employ', u'protect'),\n",
       " (u'build', u'tope'),\n",
       " (u'prepare', u'supply'),\n",
       " (u'caparison', u'elephant'),\n",
       " (u'present', u'flowers'),\n",
       " (u'present', u'incense'),\n",
       " (u'turn', u'furrow'),\n",
       " (u'promise', u'command'),\n",
       " (u'cast', u'forth'),\n",
       " (u'acknowledge', u'superior'),\n",
       " (u'seek', u'alliance'),\n",
       " (u'offer', u'price'),\n",
       " (u'rest', u'responsibility'),\n",
       " (u'undertake', u'repair'),\n",
       " (u'command', u'celebration'),\n",
       " (u'detest', u'religion'),\n",
       " (u'prove', u'faithful'),\n",
       " (u'betroth', u'daughters'),\n",
       " (u'raise', u'siege'),\n",
       " (u'provide', u'vessels'),\n",
       " (u'summon', u'force'),\n",
       " (u'stake', u'weight'),\n",
       " (u'cause', u'sick'),\n",
       " (u'cause', u'wounded'),\n",
       " (u'cede', u'conquests'),\n",
       " (u'allow', u'suspicion'),\n",
       " (u'allow', u'cruelty'),\n",
       " (u'open', u'gates'),\n",
       " (u'order', u'proceeded'),\n",
       " (u'order', u'appeared'),\n",
       " (u'declare', u'war'),\n",
       " (u'recall', u'contingents'),\n",
       " (u'push', u'officiousness'),\n",
       " (u'accept', u'award'),\n",
       " (u'tolerate', u'round'),\n",
       " (u'march', u'army'),\n",
       " (u'grant', u'peace'),\n",
       " (u'create', u'member'),\n",
       " (u'create', u'life'),\n",
       " (u'devote', u'day'),\n",
       " (u'tap', u'hold'),\n",
       " (u'order', u'daughter'),\n",
       " (u'order', u'infant'),\n",
       " (u'order', u'cast'),\n",
       " (u'die', u'927'),\n",
       " (u'decline', u'purchase'),\n",
       " (u'appoint', u'successor'),\n",
       " (u'claim', u'right'),\n",
       " (u'exercise', u'right'),\n",
       " (u'turn', u'foe'),\n",
       " (u'cut', u'boots'),\n",
       " (u'enter', u'dance'),\n",
       " (u'owe', u'throne'),\n",
       " (u'draw', u'wand'),\n",
       " (u'kill', u'assassin'),\n",
       " (u'promise', u'crown'),\n",
       " (u'cut', u'boots'),\n",
       " (u'order', u'body'),\n",
       " (u'picture', u'trophies'),\n",
       " (u'attend', u'races'),\n",
       " (u'grant', u'licence'),\n",
       " (u'enjoy', u'rest'),\n",
       " (u'reflect', u'refused'),\n",
       " (u'reflect', u'told'),\n",
       " (u'address', u'brother'),\n",
       " (u'address', u'speech'),\n",
       " (u'enter', u'arena'),\n",
       " (u'squeeze', u'thumbs'),\n",
       " (u'address', u'queen'),\n",
       " (u'draw', u'sigh'),\n",
       " (u'salute', u'subjects'),\n",
       " (u'commit', u'act'),\n",
       " (u'strike', u'person'),\n",
       " (u'present', u'name'),\n",
       " (u'instruct', u'ambassador'),\n",
       " (u'borrow', u'copies'),\n",
       " (u'command', u'canoes'),\n",
       " (u'issue', u'decree'),\n",
       " (u'secure', u'condemnation'),\n",
       " (u'confirm', u'appointment'),\n",
       " (u'direct', u'judges'),\n",
       " (u'divide', u'land'),\n",
       " (u'wipe', u'lips'),\n",
       " (u'attack', u'dragon'),\n",
       " (u'lay', u'present'),\n",
       " (u'lavish', u'gifts'),\n",
       " (u'lavish', u'largesse'),\n",
       " (u'bestow', u'robes'),\n",
       " (u'enter', u'bath'),\n",
       " (u'bestow', u'robes'),\n",
       " (u'inherit', u'riches'),\n",
       " (u'add', u'jewel'),\n",
       " (u'overthrow', u'rider'),\n",
       " (u'overthrow', u'horse'),\n",
       " (u'seek', u'sage'),\n",
       " (u'charge', u'us'),\n",
       " (u'bid', u'toil'),\n",
       " (u'throw', u'handkerchief'),\n",
       " (u'propose', u'contest'),\n",
       " (u'view', u'world'),\n",
       " (u'march', u'forces'),\n",
       " (u'seek', u'relief'),\n",
       " (u'desire', u'\\nintroduced'),\n",
       " (u'answer', u'animated'),\n",
       " (u'desire', u'number'),\n",
       " (u'taketh', u'pleasure'),\n",
       " (u'face', u'students'),\n",
       " (u'findeth', u'fault'),\n",
       " (u'pardon', u'servant'),\n",
       " (u'grant', u'petition'),\n",
       " (u'disappear', u'\\ngive'),\n",
       " (u'reply', u'officer'),\n",
       " (u'reply', u'delighted'),\n",
       " (u'requireth', u'sincerity'),\n",
       " (u'turn', u'eyes'),\n",
       " (u'lead', u'legions'),\n",
       " (u'lead', u'forces'),\n",
       " (u'rehearse', u'dream'),\n",
       " (u'demand', u'honesty'),\n",
       " (u'utter', u'laugh'),\n",
       " (u'approach', u'spot'),\n",
       " (u'resume', u'duties'),\n",
       " (u'lead', u'life'),\n",
       " (u'meet', u'wishes'),\n",
       " (u'cry', u'mother'),\n",
       " (u'accept', u'gratitude'),\n",
       " (u'believe', u'man'),\n",
       " (u'enter', u'chariot'),\n",
       " (u'pass', u'hours'),\n",
       " (u'understandeth', u'servant'),\n",
       " (u'enable', u'associates'),\n",
       " (u'express', u'remorse'),\n",
       " (u'order', u'guards'),\n",
       " (u'change', u'color'),\n",
       " (u'cause', u'\\nconveyed'),\n",
       " (u'marry', u'sister'),\n",
       " (u'confide', u'power'),\n",
       " (u'order', u'boats'),\n",
       " (u'beg', u'piece'),\n",
       " (u'fit', u'boats'),\n",
       " (u'serve', u'guests'),\n",
       " (u'receive', u'us'),\n",
       " (u'receive', u'kindly'),\n",
       " (u'cut', u'open'),\n",
       " (u'cut', u'company'),\n",
       " (u'return', u'letter'),\n",
       " (u'reside', u'land'),\n",
       " (u'sendeth', u'letter'),\n",
       " (u'die', u'day'),\n",
       " (u'command', u'people'),\n",
       " (u'place', u'us'),\n",
       " (u'interrupt', u'formality'),\n",
       " (u'embrace', u'head'),\n",
       " (u'understand', u'mistake'),\n",
       " (u'command', u'prince'),\n",
       " (u'offer', u'lack'),\n",
       " (u'desire', u'son'),\n",
       " (u'desire', u'\\ntouch'),\n",
       " (u'command', u'nephew'),\n",
       " (u'bow', u'head'),\n",
       " (u'cause', u'sluice'),\n",
       " (u'sit', u'hear'),\n",
       " (u'offer', u'\\nwear'),\n",
       " (u'alter', u'plans'),\n",
       " (u'foster', u'division'),\n",
       " (u'foster', u'emulation'),\n",
       " (u'receive', u'letter'),\n",
       " (u'descend', u'stairs'),\n",
       " (u'dip', u'finger'),\n",
       " (u'wear', u'turban'),\n",
       " (u'command', u'son'),\n",
       " (u'rest', u'days'),\n",
       " (u'cause', u'town'),\n",
       " (u'remove', u'camp'),\n",
       " (u'pitch', u'tent'),\n",
       " (u'conceal', u'opinion'),\n",
       " (u'cause', u'lists'),\n",
       " (u'pay', u'compliments'),\n",
       " (u'command', u'presence'),\n",
       " (u'allow', u'females'),\n",
       " (u'place', u'confidence'),\n",
       " (u'remove', u'miles'),\n",
       " (u'turn', u'tune'),\n",
       " (u'change', u'manner'),\n",
       " (u'order', u'convert'),\n",
       " (u'announce', u'adherence'),\n",
       " (u'carry', u'zeal'),\n",
       " (u'name', u'etymologies'),\n",
       " (u'bear', u'-than'),\n",
       " (u'allow', u'case'),\n",
       " (u'fear', u'propaganda'),\n",
       " (u'solve', u'riddle'),\n",
       " (u'court', u'death'),\n",
       " (u'order', u'general'),\n",
       " (u'order', u'retreat'),\n",
       " (u'devour', u'oyster'),\n",
       " (u'enable', u'despatch'),\n",
       " (u'exercise', u'power'),\n",
       " (u'issue', u'proclamation'),\n",
       " (u'publish', u'proclamation'),\n",
       " (u'assume', u'\\ncommand'),\n",
       " (u'press', u'invitation'),\n",
       " (u'address', u'letter'),\n",
       " (u'pay', u'visit'),\n",
       " (u'recommend', u'mercy'),\n",
       " (u'visit', u'hareem'),\n",
       " (u'shave', u'head'),\n",
       " (u'eat', u'\\nmeat'),\n",
       " (u'command', u'us'),\n",
       " (u'command', u'light'),\n",
       " (u'sustain', u'us'),\n",
       " (u'advise', u'\\n    spurn'),\n",
       " (u'recover', u'throne'),\n",
       " (u'bid', u'officers'),\n",
       " (u'enter', u'orchard'),\n",
       " (u'change', u'plan'),\n",
       " (u'follow', u'advice'),\n",
       " (u'order', u'chains'),\n",
       " (u'publish', u'decrees'),\n",
       " (u'forbid', u'war'),\n",
       " (u'grant', u'point'),\n",
       " (u'grant', u'point'),\n",
       " (u'enjoy', u'rights'),\n",
       " (u'remain', u'friend'),\n",
       " (u'die', u'feast'),\n",
       " (u'owe', u'crown'),\n",
       " (u'obey', u'person'),\n",
       " (u'surrender', u'pretence'),\n",
       " (u'infuse', u'vivacity'),\n",
       " (u'infuse', u'vigor'),\n",
       " (u'need', u'retinue'),\n",
       " (u'need', u'bodyguard'),\n",
       " (u'need', u'host'),\n",
       " (u'think', u'maiden'),\n",
       " (u'order', u'magicians'),\n",
       " (u'order', u'\\nappear'),\n",
       " (u'order', u'nobles'),\n",
       " (u'describe', u'expedition'),\n",
       " (u'write', u'letter'),\n",
       " (u'fight', u'battle'),\n",
       " (u'dismiss', u'children'),\n",
       " (u'cause', u'silver'),\n",
       " (u'cause', u'gold'),\n",
       " (u'loveth', u'things'),\n",
       " (u'cause', u'men'),\n",
       " (u'cause', u'bowmen'),\n",
       " (u'curse', u'times'),\n",
       " (u'proclaim', u'feast'),\n",
       " (u'enter', u'temple'),\n",
       " (u'close', u'doors'),\n",
       " (u'dress', u'figure'),\n",
       " (u'admire', u'last'),\n",
       " (u'search', u'land'),\n",
       " (u'form', u'story'),\n",
       " (u'lead', u'company'),\n",
       " (u'answer', u'etc'),\n",
       " (u'apply', u'word'),\n",
       " (u'grow', u'mid'),\n",
       " (u'break', u'hearts'),\n",
       " (u'rend', u'doors'),\n",
       " (u'cede', u'kingdom'),\n",
       " (u'force', u'monks'),\n",
       " (u'threaten', u'merchants'),\n",
       " (u'rule', u'kingdom'),\n",
       " (u'fight', u'quarrels'),\n",
       " (u'fight', u'duel'),\n",
       " (u'throw', u'warder'),\n",
       " (u'examine', u'said'),\n",
       " (u'din', u'wood'),\n",
       " (u'settle', u'pension'),\n",
       " (u'withdraw', u'hand'),\n",
       " (u'set', u'foot'),\n",
       " (u'size', u'situation'),\n",
       " (u'strike', u'hand'),\n",
       " (u'groan', u'aloud'),\n",
       " (u'turn', u'head'),\n",
       " (u'manage', u'pass'),\n",
       " (u'lose', u'breath'),\n",
       " (u'pick', u'stone'),\n",
       " (u'pick', u'stone'),\n",
       " (u'urge', u'leave'),\n",
       " (u'toss', u'jewels'),\n",
       " (u'summon', u'council'),\n",
       " (u'ordain', u'festival'),\n",
       " (u'live', u'years'),\n",
       " (u'appoint', u'feast'),\n",
       " (u'stand', u'tree'),\n",
       " (u'promise', u'life'),\n",
       " (u'order', u'person'),\n",
       " (u'think', u'fit'),\n",
       " (u'drink', u'wine'),\n",
       " (u'appoint', u'governor'),\n",
       " (u'bestow', u'governorships'),\n",
       " (u'ascend', u'throne'),\n",
       " (u'order', u'us'),\n",
       " (u'order', u'man'),\n",
       " (u'promise', u'\\ndismiss'),\n",
       " (u'demand', u'surrender'),\n",
       " (u'levy', u'troops'),\n",
       " (u'refuse', u'consent'),\n",
       " (u'consult', u'council'),\n",
       " (u'judge', u'prudent'),\n",
       " (u'utter', u'order'),\n",
       " (u'stroke', u'beard'),\n",
       " (u'notice', u'seats'),\n",
       " (u'allow', u'slices'),\n",
       " (u'receive', u'reward'),\n",
       " (u'characterize', u'course'),\n",
       " (u'form', u'counterblast'),\n",
       " (u'dispose', u'men'),\n",
       " (u'grant', u'confirmation'),\n",
       " (u'grant', u'protection'),\n",
       " (u'express', u'agreement'),\n",
       " (u'kiss', u'knights'),\n",
       " (u'appoint', u'defenders'),\n",
       " (u'appoint', u'man'),\n",
       " (u'summon', u'levy'),\n",
       " (u'offer', u'money'),\n",
       " (u'declare', u'opposed'),\n",
       " (u'fold', u'sheet'),\n",
       " (u'claim', u'victory'),\n",
       " (u'claim', u'seem'),\n",
       " (u'obtain', u'victory'),\n",
       " (u'adopt', u'names'),\n",
       " (u'adopt', u'titles'),\n",
       " (u'claim', u'origin'),\n",
       " (u'quash', u'decree'),\n",
       " (u'receive', u'summons'),\n",
       " (u'succeed', u'father'),\n",
       " (u'dispossess', u'house'),\n",
       " (u'occasion', u'letter'),\n",
       " (u'accept', u'baptism'),\n",
       " (u'furnish', u'preface'),\n",
       " (u'bear', u'name'),\n",
       " (u'carry', u'battalions'),\n",
       " (u'assume', u'control'),\n",
       " (u'adopt', u'name'),\n",
       " (u'join', u'allies'),\n",
       " (u'claim', u'precedence'),\n",
       " (u'appoint', u'premier'),\n",
       " (u'succeed', u'nephew'),\n",
       " (u'expose', u'daughter'),\n",
       " (u'inform', u'officers'),\n",
       " (u'set', u'number'),\n",
       " (u'receive', u'end'),\n",
       " (u'endorse', u'recommendations'),\n",
       " (u'extend', u'dominions'),\n",
       " (u'extend', u'farthest'),\n",
       " (u'sit', u'silent'),\n",
       " (u'sit', u'perplexed'),\n",
       " (u'order', u'two'),\n",
       " (u'order', u'three'),\n",
       " (u'dismiss', u'complaints'),\n",
       " (u'provoke', u'206'),\n",
       " (u'grant', u'settlers'),\n",
       " (u'notwithstand', u'acknowledged'),\n",
       " (u'notwithstand', u'renunciation'),\n",
       " (u'refuse', u'\\nlisten'),\n",
       " (u'refuse', u'authorize'),\n",
       " (u'assume', u'powers'),\n",
       " (u'indicate', u'neutrality'),\n",
       " (u'treat', u'subjects'),\n",
       " (u'address', u'years'),\n",
       " (u'end', u'letter'),\n",
       " (u'impose', u'costs'),\n",
       " (u'promise', u'favour'),\n",
       " (u'tax', u'colonies'),\n",
       " (u'collect', u'revenue'),\n",
       " (u'regret', u'repeal'),\n",
       " (u'seize', u'papers'),\n",
       " (u'express', u'regret'),\n",
       " (u'change', u'constitution'),\n",
       " (u'require', u'appropriations'),\n",
       " (u'refuse', u'\\nsee'),\n",
       " (u'propose', u'continue'),\n",
       " (u'deny', u'audience'),\n",
       " (u'promise', u'deny'),\n",
       " (u'choose', u'monkey'),\n",
       " (u'resume', u'power'),\n",
       " (u'open', u'session'),\n",
       " (u'understand', u'value'),\n",
       " (u'lie', u'treasures'),\n",
       " (u'lay', u'treasures'),\n",
       " (u'lay', u'treasures'),\n",
       " (u'fix', u'eye'),\n",
       " (u'summon', u'whenever'),\n",
       " (u'force', u'furthermore'),\n",
       " (u'issue', u'proclamation'),\n",
       " (u'produce', u'letters'),\n",
       " (u'grant', u'lands'),\n",
       " (u'invite', u'lady'),\n",
       " (u'turn', u'backs'),\n",
       " (u'present', u'abbey'),\n",
       " (u'excel', u'rest'),\n",
       " (u'name', u'cousin'),\n",
       " (u'lie', u'hid'),\n",
       " (u'kill', u'sons'),\n",
       " (u'grow', u'pale'),\n",
       " (u'command', u'\\nbrought'),\n",
       " (u'command', u'thrown'),\n",
       " (u'scale', u'wall'),\n",
       " (u'cry', u'chairmen'),\n",
       " (u'traverse', u'poorest'),\n",
       " (u'traverse', u'quarters'),\n",
       " (u'run', u'round'),\n",
       " (u'inform', u'us'),\n",
       " (u'open', u'chapter'),\n",
       " (u'strike', u'demon'),\n",
       " (u'enter', u'gate'),\n",
       " (u'address', u'sage'),\n",
       " (u'restore', u'lord'),\n",
       " (u'resign', u'care'),\n",
       " (u'enjoy', u'repose'),\n",
       " (u'address', u'lord'),\n",
       " (u'cheer', u'friend'),\n",
       " (u'look', u'silent'),\n",
       " (u'address', u'priest'),\n",
       " (u'enter', u'room'),\n",
       " (u'distract', u'lies'),\n",
       " (u'stand', u'mother'),\n",
       " (u'stand', u'dear'),\n",
       " (u'renew', u'lament'),\n",
       " (u'repeat', u\"o'er\"),\n",
       " (u'kiss', u'head'),\n",
       " (u'seek', u'skies'),\n",
       " (u'deem', u'man'),\n",
       " (u'gave', u'boons'),\n",
       " (u'draw', u'host'),\n",
       " (u'draw', u'retinue'),\n",
       " (u'set', u'eyes'),\n",
       " (u'secure', u'fame'),\n",
       " (u'vouchsafe', u'aid'),\n",
       " (u'resign', u'car'),\n",
       " (u'strive', u'join'),\n",
       " (u'deny', u'subjects'),\n",
       " (u'forsake', u'friend'),\n",
       " (u'forsake', u'plighted'),\n",
       " (u'forsake', u'break'),\n",
       " (u'clasp', u'feet'),\n",
       " (u'review', u'multitude'),\n",
       " (u'cleave', u'sky'),\n",
       " (u'allot', u'us'),\n",
       " (u'spy', u'princes'),\n",
       " (u'lift', u'us'),\n",
       " (u'lift', u'high'),\n",
       " (u'turn', u'eyes'),\n",
       " (u'greet', u'fair'),\n",
       " (u'bow', u'head'),\n",
       " (u'approach', u'dame'),\n",
       " (u'return', u'eyes'),\n",
       " (u'press', u'fingers'),\n",
       " (u'enjoy', u'lives'),\n",
       " (u'confer', u'immortality'),\n",
       " (u'possess', u'authority'),\n",
       " (u'possess', u'authority'),\n",
       " (u'accompany', u'renewed'),\n",
       " (u'exhibit', u'favoritism'),\n",
       " (u'carry', u'aversion'),\n",
       " (u'deserve', u'credit'),\n",
       " (u'blow', u'note'),\n",
       " (u'present', u'letters'),\n",
       " (u'read', u'letters'),\n",
       " (u'read', u'anger'),\n",
       " (u'assemble', u'goodly'),\n",
       " (u'raise', u'head'),\n",
       " (u'draw', u'sword'),\n",
       " (u'blow', u'note'),\n",
       " (u'realize', u'wisdom'),\n",
       " (u'shake', u'head'),\n",
       " (u'allow', u'\\nattempt'),\n",
       " (u'blow', u'horn'),\n",
       " (u'understand', u'little'),\n",
       " (u'return', u'answer'),\n",
       " (u'seek', u'aid'),\n",
       " (u'adopt', u'dubs'),\n",
       " (u'defy', u'idols'),\n",
       " (u'stir', u'strife'),\n",
       " (u'horn', u'returns'),\n",
       " (u'blind', u\"e'e\"),\n",
       " (u'nobl', u'strove'),\n",
       " (u'receive', u'partridges'),\n",
       " (u'command', u'officers'),\n",
       " (u'command', u'fetch'),\n",
       " (u'fall', u'asleep'),\n",
       " (u'wonder', u'much'),\n",
       " (u'grow', u'ould'),\n",
       " (u'worship', u'crystal'),\n",
       " (u'deserve', u'life'),\n",
       " (u'lose', u'either'),\n",
       " (u'let', u'purvey'),\n",
       " (u'let', u'let'),\n",
       " (u'fall', u'asleep'),\n",
       " (u'weave', u'reeds'),\n",
       " (u'lift', u'hand'),\n",
       " (u'bow', u'head'),\n",
       " (u'deride', u'ceremonies'),\n",
       " (u'lose', u'crown'),\n",
       " (u'lose', u'life'),\n",
       " (u'pasture', u'elephants'),\n",
       " (u'pasture', u'steeds'),\n",
       " (u'answer', u'question'),\n",
       " (u'publish', u'edict'),\n",
       " (u'announce', u'decision'),\n",
       " (u'play', u'whist'),\n",
       " (u'countermand', u'orders'),\n",
       " (u'conceal', u'dislike'),\n",
       " (u'prefer', u'\\ndeliver'),\n",
       " (u'recall', u'army'),\n",
       " (u'accept', u'resignation'),\n",
       " (u'write', u'pledge'),\n",
       " (u'cause', u'vacancy'),\n",
       " (u'establish', u'government'),\n",
       " (u'permit', u'\\nreplaced'),\n",
       " (u'assert', u'\\nmade'),\n",
       " (u'announce', u'intention'),\n",
       " (u'issue', u'rescript'),\n",
       " (u'terminate', u'audience'),\n",
       " (u'sign', u'agreement'),\n",
       " (u'revive', u'dignity'),\n",
       " (u'write', u'letters'),\n",
       " (u'spake', u'words'),\n",
       " (u'accept', u'gift'),\n",
       " (u'pay', u'servant'),\n",
       " (u'pay', u'tribute'),\n",
       " (u'lay', u'package'),\n",
       " (u'summon', u'number'),\n",
       " (u'join', u'men'),\n",
       " (u'dismiss', u'ambassador'),\n",
       " (u'issue', u'orders'),\n",
       " (u'bear', u'affection'),\n",
       " (u'assume', u'power'),\n",
       " (u'sell', u'right'),\n",
       " (u'issue', u'edict'),\n",
       " (u'throw', u'open'),\n",
       " (u'summon', u'court'),\n",
       " (u'wear', u'head'),\n",
       " (u'marry', u'daughter'),\n",
       " (u'form', u'link'),\n",
       " (u'present', u'spectacle'),\n",
       " (u'decree', u'pain'),\n",
       " (u'serve', u'men'),\n",
       " (u'tear', u'hairs'),\n",
       " (u'offer', u'sacrifice'),\n",
       " (u'order', u'number'),\n",
       " (u'set', u'foot'),\n",
       " (u'shake', u'head'),\n",
       " (u'bow', u'head'),\n",
       " (u'bid', u'go'),\n",
       " (u'declare', u'holiday'),\n",
       " (u'draw', u'nearer'),\n",
       " (u'count', u'money'),\n",
       " (u'order', u'huntsmen'),\n",
       " (u'order', u'huntsmen'),\n",
       " (u'open', u'chamber'),\n",
       " (u'borrow', u'money'),\n",
       " (u'cast', u'eyes'),\n",
       " (u'shew', u'letter'),\n",
       " (u'commence', u'mission'),\n",
       " (u'disregard', u'authority'),\n",
       " (u'lose', u'xxiii'),\n",
       " (u'kill', u'kingly'),\n",
       " (u'write', u'discharge'),\n",
       " (u'kill', u'deer'),\n",
       " (u'profess', u'friendship'),\n",
       " (u'beg', u'life'),\n",
       " (u'occupy', u'seat'),\n",
       " (u'sleep', u'nights'),\n",
       " (u'place', u'son'),\n",
       " (u'reprove', u'preceptor'),\n",
       " (u'distribute', u'honor'),\n",
       " (u'distribute', u'right'),\n",
       " (u'distribute', u'left'),\n",
       " (u'accept', u'seat'),\n",
       " (u'consult', u'proprietor'),\n",
       " (u'turn', u'corner'),\n",
       " (u'enter', u'room'),\n",
       " (u'idle', u'hours'),\n",
       " (u'open', u'note'),\n",
       " (u'draw', u'simple'),\n",
       " (u'open', u'windows'),\n",
       " (u'enter', u'lobby'),\n",
       " (u'enter', u'office'),\n",
       " (u'praise', u'courage'),\n",
       " (u'praise', u'patriotism'),\n",
       " (u'behold', u'armies'),\n",
       " (u'clove', u'skull'),\n",
       " (u'break', u'spear'),\n",
       " (u'plot', u'life'),\n",
       " (u'fall', u'heir'),\n",
       " (u'rout', u'army'),\n",
       " (u'pass', u'hours'),\n",
       " (u'order', u'sister'),\n",
       " (u'recognize', u'wife'),\n",
       " (u'occupy', u'throne'),\n",
       " (u'receive', u'petitions'),\n",
       " (u'lift', u'hand'),\n",
       " (u'duck', u'head'),\n",
       " (u'grant', u'control'),\n",
       " (u'promise', u'payment'),\n",
       " (u'retain', u'power'),\n",
       " (u'confiscate', u'property'),\n",
       " (u'outlaw', u'clergy'),\n",
       " (u'sing', u'locks'),\n",
       " (u'avoid', u'colour'),\n",
       " (u'avoid', u'pretext'),\n",
       " (u'play', u'cards'),\n",
       " (u'lose', u'submission'),\n",
       " (u'lose', u'reverence'),\n",
       " (u'cut', u'knot'),\n",
       " (u'cut', u'married.[125'),\n",
       " (u'demand', u'sentence'),\n",
       " (u'disobey', u'brief'),\n",
       " (u'thenceforward', u'regard'),\n",
       " (u'suspend', u'resolution'),\n",
       " (u'need', u'help'),\n",
       " (u'advance', u'money'),\n",
       " (u'receive', u'due'),\n",
       " (u'set', u'evil'),\n",
       " (u'set', u'example'),\n",
       " (u'refuse', u'satisfaction'),\n",
       " (u'reconsider', u'position'),\n",
       " (u'present', u'request'),\n",
       " (u'reply', u'declared'),\n",
       " (u'pay', u'share'),\n",
       " (u'assign', u'crown'),\n",
       " (u'reply', u'\\nrequire'),\n",
       " (u'end', u'conversation'),\n",
       " (u'prefer', u'friendship'),\n",
       " (u'decide', u'marry'),\n",
       " (u'rise', u';'),\n",
       " (u'depart', u'life'),\n",
       " (u'lay', u'head'),\n",
       " (u'desire', u'reconciliation'),\n",
       " (u'pay', u'attentions'),\n",
       " (u'receive', u'letter'),\n",
       " (u'modify', u'demand'),\n",
       " (u'touch', u'destruction'),\n",
       " (u'drink', u'quantity'),\n",
       " (u'happen', u'struck'),\n",
       " (u'guarantee', u'alliance'),\n",
       " (u'conceive', u'plan'),\n",
       " (u'stain', u'honour'),\n",
       " (u'draw', u'rubles'),\n",
       " (u'observe', u'change'),\n",
       " (u'condescend', u'held'),\n",
       " (u'condescend', u'rolled'),\n",
       " (u'condescend', u'began'),\n",
       " (u'grant', u'meadow'),\n",
       " (u'shake', u'hands'),\n",
       " (u'snuff', u'wick'),\n",
       " (u'lean', u'body'),\n",
       " (u'observe', u'clergyman'),\n",
       " (u'evince', u'interest'),\n",
       " (u'punish', u'villains'),\n",
       " (u'lead', u'expedition'),\n",
       " (u'grant', u'audience'),\n",
       " (u'cause', u'man'),\n",
       " (u'observe', u'ceremony'),\n",
       " (u'order', u'people'),\n",
       " (u'bury', u'forty'),\n",
       " (u'murder', u'wife'),\n",
       " (u'murder', u'smith'),\n",
       " (u'murder', u'children'),\n",
       " (u'win', u'admiration'),\n",
       " (u'like', u'place'),\n",
       " (u'demand', u'name'),\n",
       " (u'persuade', u'people'),\n",
       " (u'trace', u'story'),\n",
       " (u'receive', u'lady'),\n",
       " (u'promise', u'pardon'),\n",
       " (u'pay', u'visits'),\n",
       " (u'wound', u'pride'),\n",
       " (u'receive', u'prince'),\n",
       " (u'cause', u'inquiry'),\n",
       " (u'pledge', u'\\nunite'),\n",
       " (u'carry', u'conquest'),\n",
       " (u'form', u'ministry'),\n",
       " (u'express', u'joy'),\n",
       " (u'express', u'gratitude'),\n",
       " (u'carry', u'affection'),\n",
       " (u'enter', u'capital'),\n",
       " (u'rid', u'horseback'),\n",
       " (u'open', u'heart'),\n",
       " (u'rear', u'monument'),\n",
       " (u'perform', u'prodigies'),\n",
       " (u'manifest', u'indignation'),\n",
       " (u'enter', u'barrack'),\n",
       " (u'sign', u'treaty'),\n",
       " (u'join', u'enemies'),\n",
       " (u'age', u'years'),\n",
       " (u'review', u'pieces'),\n",
       " (u'govern', u'bodies--_we'),\n",
       " (u'conduct', u'business'),\n",
       " (u'think', u'moment'),\n",
       " (u'beckon', u'approach'),\n",
       " (u'cause', u'day'),\n",
       " (u'abdicate', u'throne'),\n",
       " (u'return', u'morrow'),\n",
       " (u'protest', u'feebly'),\n",
       " (u'enjoy', u'influence'),\n",
       " (u'like', u'men'),\n",
       " (u'quarter', u'us'),\n",
       " (u'like', u'truth'),\n",
       " (u'address', u'question'),\n",
       " (u'address', u'child'),\n",
       " (u'address', u'lord'),\n",
       " (u'enquire', u'cause'),\n",
       " (u'afford', u'instances'),\n",
       " (u'predict', u'fate'),\n",
       " (u'run', u'high'),\n",
       " (u'list', u'1/2d'),\n",
       " (u'dismiss', u'retinue'),\n",
       " (u'choose', u'men'),\n",
       " (u'lift', u'face'),\n",
       " (u'order', u'coat'),\n",
       " (u'raise', u'hands'),\n",
       " (u'thank', u'said'),\n",
       " (u'raise', u'hands'),\n",
       " (u'swear', u'fealty'),\n",
       " (u'wear', u'pearls'),\n",
       " (u'surpass', u'kings'),\n",
       " (u'tear', u'arms'),\n",
       " (u'build', u'house'),\n",
       " (u'issue', u'commission'),\n",
       " (u'add', u'touch'),\n",
       " (u'play', u'dominoes'),\n",
       " (u'exchange', u'glance'),\n",
       " (u'raise', u'eyes'),\n",
       " (u'remark', u'movement'),\n",
       " (u'tap', u'foot'),\n",
       " (u'occupy', u'throne'),\n",
       " (u'sit', u'throne'),\n",
       " (u'deem', u'guilty'),\n",
       " (u'display', u'impartiality'),\n",
       " (u'hint', u'suspicions'),\n",
       " (u'shake', u'head'),\n",
       " (u'bear', u'slander'),\n",
       " (u'nominate', u'members'),\n",
       " (u'impose', u'tasks'),\n",
       " (u'claim', u'honours'),\n",
       " (u'overrun', u'lands'),\n",
       " (u'engage', u'activities'),\n",
       " (u'kill', u'tigress'),\n",
       " (u'utter', u'exclamation'),\n",
       " (u'utter', u'little'),\n",
       " (u'pay', u'addresses'),\n",
       " (u'express', u'regard'),\n",
       " (u'wear', u'coat'),\n",
       " (u'visit', u'another'),\n",
       " (u'turn', u'sovereign'),\n",
       " (u'fear', u'arm'),\n",
       " (u'utter', u'exclamation'),\n",
       " (u'utter', u'little'),\n",
       " (u'name', u'generalissimo'),\n",
       " (u'employ', u'words'),\n",
       " (u'desire', u'\\n   risk'),\n",
       " (u'preside', u'hatred'),\n",
       " (u'spare', u'us'),\n",
       " (u'announce', u'reestablishment'),\n",
       " (u'fold', u'arms'),\n",
       " (u'lead', u'us'),\n",
       " (u'arrange', u'pillows'),\n",
       " (u'follow', u'path'),\n",
       " (u'shake', u'hands'),\n",
       " (u'bend', u'ear'),\n",
       " (u'shake', u'hands'),\n",
       " (u'betray', u'subject'),\n",
       " (u'assume', u'expression'),\n",
       " (u'open', u'eyes'),\n",
       " (u'raise', u'head')]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "noun2v_o[\"king\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, filter out those `(s,v,o)` combinations in which the `v` and `o` are not similar according to word2vec model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 23431/23431 [00:11<00:00, 2037.84it/s]\n"
     ]
    }
   ],
   "source": [
    "new_noun2v_o = defaultdict(list)\n",
    "\n",
    "for k in tqdm(noun2v_o.keys()):\n",
    "    vos = []\n",
    "    for verb,obj in noun2v_o[k]:\n",
    "        try:\n",
    "            vos.append((verb,obj,model.similarity(obj,verb)))\n",
    "        except:\n",
    "            pass\n",
    "    vos.sort(key = lambda x: x[2], reverse=True)\n",
    "    \n",
    "    ##again, logic to handle rare and common nouns differently\n",
    "    if len(vos)>20:\n",
    "        for verb,obj,value in vos[:10]:\n",
    "            new_noun2v_o[k].append((verb,obj))\n",
    "    elif len(vos)>10:\n",
    "        for verb,obj,value in vos[:5]:\n",
    "            new_noun2v_o[k].append((verb,obj))\n",
    "    elif len(vos)>2:\n",
    "        verb,obj,value = vos[0]\n",
    "        new_noun2v_o[k].append((verb,obj))\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'produce', u'plant'),\n",
       " (u'bruise', u'head'),\n",
       " (u'burst', u'breaks'),\n",
       " (u'produce', u'fruits'),\n",
       " (u'prove', u'none'),\n",
       " (u'bear', u'fruit'),\n",
       " (u'lie', u'latent'),\n",
       " (u'beg', u'bread'),\n",
       " (u'contain', u'tannin'),\n",
       " (u'contain', u'mixed')]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_noun2v_o[\"seed\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"data/noun2v_o.json\",\"w\") as f:\n",
    "    json.dump(new_noun2v_o,f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}