Simple_Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from Utils.FS import file\n",
    "from nltk.tokenize.punkt import PunktSentenceTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import brown\n",
    "from scipy.sparse import coo_matrix, dok_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from sklearn.preprocessing import normalize\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from sklearn.manifold import TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1234)\n",
    "WINDOW_SIZE = 5\n",
    "NORM = 'l2'\n",
    "SVD_DIMENSION = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sents = brown.sents()\n",
    "sents = [[token.lower() for token in sent] for sent in sents]\n",
    "words = brown.words()\n",
    "words = [word.lower() for word in words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(\"Number of tokens: {}\".format(len(words)))\n",
    "print(\"Number of sentences: {}\".format(len(sents)))\n",
    "print(\"Longest sentences length: {}\".format(max([len(sent) for sent in sents])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def words2dicts(words):\n",
    "    f_dict = {}\n",
    "    index = 0\n",
    "    for word in words:\n",
    "        if not word in f_dict.keys():\n",
    "            f_dict[word] = index\n",
    "            index = index + 1\n",
    "        \n",
    "    return f_dict, { v:k for k, v in f_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "words_dict, inv_words_dict = words2dicts(words)\n",
    "words_size = len(words_dict)\n",
    "print(\"Number of unique tokens: {}\".format(words_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sents2wordContextMatrix(sents, words_dict, window = 5):\n",
    "    m = coo_matrix((words_size, words_size), 'float64')\n",
    "    \n",
    "    print('Doing forward pass...')\n",
    "    back = []\n",
    "    row = []\n",
    "    col = []\n",
    "    data = []\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            row += [words_dict[word] for b in back]\n",
    "            col += [words_dict[b] for b in back]\n",
    "            data += [1 for b in back]\n",
    "            if len(back) == window:\n",
    "                back.pop(0)\n",
    "            back.append(word)\n",
    "    \n",
    "    print('Doing backward pass...')\n",
    "    back = []\n",
    "    for sent in reversed(sents):\n",
    "        for word in reversed(sent):\n",
    "            row += [words_dict[word] for b in back]\n",
    "            col += [words_dict[b] for b in back]\n",
    "            data += [1 for b in back]\n",
    "            if len(back) == window:\n",
    "                back.pop(0)\n",
    "            back.append(word)\n",
    "    \n",
    "    return coo_matrix((data, (row, col)), shape=(words_size, words_size), dtype='float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def sents2freq(sents):\n",
    "    freq = {}\n",
    "    for sent in sents:\n",
    "        for word in sent:\n",
    "            if word in freq.keys():\n",
    "                freq[word] += 1\n",
    "            else:\n",
    "                freq[word] = 1\n",
    "    return freq\n",
    "\n",
    "words_freq = sents2freq(sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "m = sents2wordContextMatrix(sents, words_dict, WINDOW_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "m.tocsr()[0].todense()[0, 0]\n",
    "inv_words_dict[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "norm_m = normalize(m, norm=NORM, axis=1, copy=True, return_norm=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "u, s, vt = svds(norm_m, k=SVD_DIMENSION)\n",
    "#u, s, vt = svds(m, k=SVD_DIMENSION)\n",
    "\n",
    "#norm_u = normalize(u, norm=NORM, axis=1, copy=True, return_norm=False)\n",
    "\n",
    "words_vec = {}\n",
    "for i in range(u.shape[0]):\n",
    "    words_vec[inv_words_dict[i]] = u[i]\n",
    "print(words_vec['man'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotData(vocabs, X, Y):\n",
    "    plt.scatter(X, Y)\n",
    "    plt.axis([min(X), max(X), min(Y), max(Y)])\n",
    "    for label, x, y in zip(vocabs, X, Y):\n",
    "        plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')\n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot(vocabs, words_vec):\n",
    "    X = [words_vec[vocab][0] for vocab in vocabs]\n",
    "    Y = [words_vec[vocab][1] for vocab in vocabs]\n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plotTSNE(vocabs, words_vec):\n",
    "    tsne = TSNE(n_components=2, random_state=0)\n",
    "    #np.set_printoptions(suppress=True)\n",
    "    data = np.array([words_vec[vocab] for vocab in vocabs])\n",
    "    print(data.shape)\n",
    "    DATA = tsne.fit_transform(data)\n",
    "    X = DATA[:, 0]\n",
    "    Y = DATA[:, 1]\n",
    "    \n",
    "    plotData(vocabs, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "vocabs = ['man', 'woman', 'king', 'queen', 'male', 'female', 'boy', 'girl']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "plot(vocabs, words_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from DataLoader import GloVe\n",
    "\n",
    "glove = GloVe.load2('./data/GloVe/glove.6B.50d.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "vocabs = ['man', 'woman', 'king', 'queen', 'male', 'female', 'boy', 'girl']\n",
    "plot(vocabs, glove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "plotTSNE(vocabs, glove)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}