NGram_Talk.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import brown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Functions we'll need for text generation</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Add open and close sentence tags\n",
    "def addS(text):\n",
    "\tfor i in range(len(text)):\n",
    "\t\ttext[i].append(u'</s>')\n",
    "\t\ttext[i].insert(0, u'<s>')\n",
    "\n",
    "\treturn text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create list of all bigrams\n",
    "def createBGList(text):\n",
    "\tBGList = []\n",
    "\tfor sent in text:\n",
    "\t\tfor word in range(1, len(sent)):\n",
    "\t\t\tBGList.append([sent[word - 1], sent[word]])\n",
    "\n",
    "\treturn BGList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create list of all trigrams\n",
    "def createTGList(text):\n",
    "\tTGList = []\n",
    "\tfor sent in text:\n",
    "\t\tfor word in range(2, len(sent)):\n",
    "\t\t\tTGList.append([[sent[word - 2], sent[word - 1]], sent[word]])\n",
    "\n",
    "\treturn TGList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Make everything lowercase\n",
    "def lowerAll(text):\n",
    "\tfor sent in range(len(text)):\n",
    "\t\tfor word in range(len(text[sent])):\n",
    "\t\t\ttext[sent][word] = text[sent][word].lower()\n",
    "\n",
    "\treturn text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# List only bigrams that start with seed word\n",
    "# Randomize list\n",
    "def nextWordBi(seed, BGs):\n",
    "    choices = [words[1] for words in BGs if words[0] == seed]\n",
    "    random.shuffle(choices)\n",
    "    \n",
    "    return choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# List only trigrams that start with seed bigram\n",
    "# Randomize list\n",
    "def nextWordTri(seed, TGs):\n",
    "    choices = [words[1] for words in TGs if words[0] == seed]\n",
    "    random.shuffle(choices)\n",
    "    \n",
    "    return choices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Get our text set up</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# For some reason this works better if I convert it to a list\n",
    "# This doesn't seem to be the case for everyone\n",
    "text = list(brown.sents())\n",
    "\n",
    "text = lowerAll(text)\n",
    "\n",
    "sText = addS(text)\n",
    "\n",
    "BGList = createBGList(sText)\n",
    "TGList = createTGList(sText)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Bigram text</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create sentence from bigrams\n",
    "seed = u'<s>'\n",
    "while seed != u'</s>':\n",
    "    word = nextWordBi(seed, BGList)\n",
    "    print word[0],\n",
    "    seed = word[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Trigram text</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create a sentence from trigrams\n",
    "# First word is chosen from bigrams\n",
    "firstWord = nextWordBi(u'<s>', BGList)\n",
    "print firstWord[0],\n",
    "seed = [u'<s>', firstWord[0]]\n",
    "while seed[1] != u'</s>':\n",
    "    word = nextWordTri(seed, TGList)\n",
    "    print word[0],\n",
    "    seed = [seed[1], word[0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Additional functions needed for spacing</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create dictionary to count frequency of each word\n",
    "def wordCount(text):\n",
    "    count = {}\n",
    "    for sent in text:\n",
    "\t\tfor word in sent:\n",
    "\t\t\tif word in count:\n",
    "\t\t\t\tcount[word] += 1\n",
    "\t\t\telse:\n",
    "\t\t\t\tcount[word] = 1\n",
    "\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create dictionary to count frequency of each bigram\n",
    "def bgCount(bgs):\n",
    "    count = {}\n",
    "    for bg in bgs:\n",
    "\t\tif bg in count:\n",
    "\t\t\tcount[bg] += 1\n",
    "\t\telse:\n",
    "\t\t\tcount[bg] = 1\n",
    "\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create list of all possible words that could follow current word\n",
    "def getPossibleWords(seed, BGs):\n",
    "    possWords = [word[1] for word in BGs if word[0] == seed]\n",
    "    \n",
    "    return possWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Insert the first space\n",
    "def spaceIt(seed, BGList, unspaced, unigramCount):\n",
    "    \n",
    "    # Create list of choices based on bigrams\n",
    "    choices = getPossibleWords(seed, BGList)\n",
    "    # Get the count of each choice\n",
    "    choiceCount = bgCount(choices)\n",
    "    topChoice = [0, 0]\n",
    "    # Loop through the unspaced string\n",
    "    # Look at every possible string (1, 12, 123, etc)\n",
    "    # If that string is one of the choices and its frequency is\n",
    "    # higher than other strings found, that is the new top choice\n",
    "    for i in range(len(unspaced)):\n",
    "        if unspaced[:i] in choices:\n",
    "            score = choiceCount[unspaced[:i]]\n",
    "            if score > topChoice[0]:\n",
    "                topChoice[0] = score\n",
    "                topChoice[1] = i\n",
    "    \n",
    "    # If no strings were found based on bigrams, back off to unigrams            \n",
    "    if topChoice[0] == 0:\n",
    "        for j in range(len(unspaced)):\n",
    "            if unspaced[:j] in unigramCount:\n",
    "                score = unigramCount[unspaced[:j]]\n",
    "                if score > topChoice[0]:\n",
    "                    topChoice[0] = score\n",
    "                    topChoice[1] = j\n",
    "    \n",
    "    return unspaced[:topChoice[1]], unspaced[topChoice[1]:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>A bit of additional setup for spacing</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unigramCount = wordCount(text)\n",
    "# A lot of single letters have a very high frequency in the corpus\n",
    "# This messes up the unigram backoff part\n",
    "# so lower the count of each to 1\n",
    "# Skip 'a' and 'i' because we actually want those counts\n",
    "letters = 'bcdefghjklmnopqurtuvwxyz'\n",
    "for letter in letters:\n",
    "    unigramCount[letter] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Get sentence from user</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "unspaced = raw_input(\"> \").lower()\n",
    "unspaced += u' '"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Time to run it!</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Starting 'seed' is open sentence tag\n",
    "seed = u'<s>'\n",
    "# Print the word the spacing function found\n",
    "# Call the spacing function on the remainder of the string\n",
    "while seed != '':\n",
    "    seed, unspaced = spaceIt(seed, BGList, unspaced, unigramCount)\n",
    "    print seed,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}