02-Visualize.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualizing Embeddings with t-SNE and Bokeh\n",
    "The tsne_bokeh function in this notebook looks overly complex with assembling data in a DataFrame and then transforming this same data into a dictionary in ColumnDataSource and then calling this data in plot_tsne.circle. Any suggestions?\n",
    "\n",
    "Inspired by [*LDA visualized using t-SNE and Bokeh*](https://www.kaggle.com/yohanb/lda-visualized-using-t-sne-and-bokeh) by Yohan, and [*Visualizing Word Vectors with t-SNE*](https://www.kaggle.com/jeffd23/visualizing-word-vectors-with-t-sne) by Jeff Delayney.\n",
    "\n",
    "## Prerequisites\n",
    "More recent versions of Bokeh are compatible with Jupyter Notebook 5 or JupyterLab, but not with Jupyter Notebook 4.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"88725010-cc19-42ea-8656-6d0a260176a8\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"88725010-cc19-42ea-8656-6d0a260176a8\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"88725010-cc19-42ea-8656-6d0a260176a8\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '88725010-cc19-42ea-8656-6d0a260176a8' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.13.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.13.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.13.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.13.0.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.13.0.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.13.0.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.13.0.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.13.0.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.13.0.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.13.0.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"88725010-cc19-42ea-8656-6d0a260176a8\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"88725010-cc19-42ea-8656-6d0a260176a8\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"88725010-cc19-42ea-8656-6d0a260176a8\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '88725010-cc19-42ea-8656-6d0a260176a8' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.13.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.13.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.13.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.13.0.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.13.0.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.13.0.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.13.0.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.13.0.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.13.0.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.13.0.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"88725010-cc19-42ea-8656-6d0a260176a8\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim') # this is only relevant in Windows.\n",
    "warnings.filterwarnings(action='ignore', category=FutureWarning, module='gensim' ) # warning will disappear in a future version of Gensim\n",
    "from gensim.models.fasttext import FastText as FT_gensim\n",
    "from sklearn.manifold import TSNE\n",
    "import pandas as pd\n",
    "from bokeh.io import output_notebook\n",
    "from bokeh.plotting import figure, show, save\n",
    "from bokeh.models import ColumnDataSource, LabelSet, Legend, LegendItem #, HoverTool, CustomJS, , Slider\n",
    "output_notebook()\n",
    "# note: more recent bokeh versions require notebook 5 or JupyterLab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the Sumerian Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_sux = FT_gensim.load(\"model/model_lemm.model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select core words\n",
    "Select between 1 and 5 words for display. Vegetables are luxury food items. The lemma `šimgig[tree]N` appears primarily in deliveries for the production of perfumes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vegetables_perfumes = [\"šimgig[tree]N\", \"hiz[vegetable]N\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find related words for each of the core words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_categories(model, words, topn=10):\n",
    "    \"\"\"word_categories takes an iterable with one or more words (up to 5) from the vocabulary of model.\n",
    "    For each word a dictionary of similar words (with the target word) is construed.\n",
    "    The value of each word is numerical (integer) and indicates the category to which it belongs.\n",
    "    The function returns a dictionary.\"\"\"\n",
    "    word_d = {}\n",
    "    for idx, word in enumerate(words):\n",
    "        w = model.wv.most_similar(word, topn=topn)\n",
    "        w = [m[0] for m in w]\n",
    "        w.append(word)\n",
    "        for item in w:\n",
    "            if item in word_d:\n",
    "                word_d[item] = 5\n",
    "            else:\n",
    "                word_d[item] = idx\n",
    "    return word_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_d = word_categories(model_sux, vegetables_perfumes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feed the Result to T-SNE and Bokeh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tsne_bokeh(model, word_d, words, fontsize=\"12pt\"):\n",
    "\n",
    "    labels = []\n",
    "    tokens = []\n",
    "    categories = []\n",
    "    legend = []\n",
    "    \n",
    "    for word in word_d:\n",
    "        tokens.append(model.wv[word])\n",
    "        labels.append(word)\n",
    "        categories.append(word_d[word])\n",
    "        if word_d[word] == 5: \n",
    "            legend.append(\"mixed\")\n",
    "        else: \n",
    "            legend.append(words[word_d[word]])\n",
    "    \n",
    "    color_d = {0: \"black\", 1: \"red\", 2: \"green\", 3: \"yellow\", 4: \"brown\", 5: \"blue\"}\n",
    "    colors = [color_d[category] for category in categories]\n",
    "    \n",
    "    tsne = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n",
    "    tsne_embedding = tsne.fit_transform(tokens)\n",
    "    tsne_embedding = pd.DataFrame(tsne_embedding, columns=['x','y'])\n",
    "    tsne_embedding[\"color\"] = colors\n",
    "    tsne_embedding[\"labels\"] = labels\n",
    "    tsne_embedding[\"legend\"] = legend\n",
    "\n",
    "    source = ColumnDataSource(\n",
    "        data=dict(\n",
    "        x = tsne_embedding.x,\n",
    "        y = tsne_embedding.y,\n",
    "        colors = tsne_embedding.color,\n",
    "        labels=tsne_embedding.labels,\n",
    "        legend = tsne_embedding.legend\n",
    "        )\n",
    "    )\n",
    "\n",
    "    l = LabelSet(x='x', y='y', text='labels', level='glyph',\n",
    "              x_offset=5, y_offset=5, source=source, render_mode='canvas', \n",
    "             text_font_size=fontsize, text_font=\"CuneiformComposite\")\n",
    "\n",
    "    plot_tsne = figure(plot_width=900, plot_height=900) #, tools=tools_tsne, title='Papers')\n",
    "    plot_tsne.circle('x', 'y', size=7, fill_color='colors', \n",
    "                  line_alpha=0, line_width=0.01, source=source, legend=\"legend\")\n",
    "    plot_tsne.add_layout(l)\n",
    "    return plot_tsne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"da16219a-d612-4023-9eed-a9966034b997\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"e5739e42-5f3f-4134-96e1-619c8b5f794e\":{\"roots\":{\"references\":[{\"attributes\":{},\"id\":\"999ee0c8-9ccd-4e63-8349-0f434637b6b6\",\"type\":\"HelpTool\"},{\"attributes\":{\"formatter\":{\"id\":\"d4d8cdaa-64d7-4bb7-92b8-6b87085e3f80\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"861993bf-063f-4619-94a5-ee249fc8130c\",\"type\":\"BasicTicker\"}},\"id\":\"f752cccb-d5f6-499a-8cf2-2398147088d9\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"cc72dcf8-65bf-485e-9a28-ab5672c68a60\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"73e2c8db-c270-4867-ad58-d86af4304189\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"ce2d32b8-0a1d-48b8-ac22-94d21c35f2ea\",\"type\":\"LinearScale\"},{\"attributes\":{\"fill_color\":{\"field\":\"colors\"},\"line_alpha\":{\"value\":0},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":0.01},\"size\":{\"units\":\"screen\",\"value\":7},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"fbdcca76-b9d3-4897-b4cf-bd30104c0c80\",\"type\":\"Circle\"},{\"attributes\":{\"plot\":{\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"73e2c8db-c270-4867-ad58-d86af4304189\",\"type\":\"BasicTicker\"}},\"id\":\"a52a964b-2ce0-4dc7-a057-0921847315bb\",\"type\":\"Grid\"},{\"attributes\":{\"formatter\":{\"id\":\"cc72dcf8-65bf-485e-9a28-ab5672c68a60\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"73e2c8db-c270-4867-ad58-d86af4304189\",\"type\":\"BasicTicker\"}},\"id\":\"d41d1699-d865-4e1e-8e19-dfd23761e2af\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"861993bf-063f-4619-94a5-ee249fc8130c\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"84d10202-c867-41ee-89cd-0c4d0fcfe859\",\"type\":\"LinearScale\"},{\"attributes\":{\"callback\":null,\"data\":{\"colors\":[\"black\",\"black\",\"black\",\"black\",\"black\",\"black\",\"black\",\"black\",\"black\",\"black\",\"black\",\"red\",\"red\",\"red\",\"red\",\"red\",\"red\",\"red\",\"red\",\"red\",\"red\",\"red\"],\"labels\":[\"gudkuru[resin]N\",\"\\u0161eli[aromatics]N\",\"\\u0161imgamgama[aromatics]N\",\"\\u0161im[aromatics]N\",\"buluh[tree]N\",\"ligidba[plant]N\",\"kukuru[resin]N\",\"arganum[conifer]N\",\"enmur[resin]N\",\"eren[cedar]N\",\"\\u0161imgig[tree]N\",\"hizurbara[vegetable]N\",\"babbarhi[plant]N\",\"HI.a[vegetable]N\",\"hazanum[plant]N\",\"kurgirina[plant]N\",\"ni\\u014bnagar[plant]N\",\"gan\\u0161e[plant]N\",\"\\u0161umsikil[vegetable]N\",\"azugna[vegetable]N\",\"ulu[plant]N\",\"hiz[vegetable]N\"],\"legend\":[\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"\\u0161imgig[tree]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\",\"hiz[vegetable]N\"],\"x\":{\"__ndarray__\":\"qk60wxOaqUOCCKVDl0ipQVuKIMRJ9ypEbbvkw6iovEJJNVLDrVWLQ03djsNKjP7DuZP+wjAJWMS2bvNDuxtFQ13MAsTMHCDDK0EaRHKR1cFb/Z/Bh8xHxA==\",\"dtype\":\"float32\",\"shape\":[22]},\"y\":{\"__ndarray__\":\"cwQHxOaRA0PlvAfEEnV7wbJWpcPFEQhDStyyQ6MfoEN62yND5sg+w4OaL8OJcjBESWtVRLKzpMHeM+xDvRcnRPBkAkJhIf1D4f13w3LPL8Tor6/DS0e7Qw==\",\"dtype\":\"float32\",\"shape\":[22]}},\"selected\":{\"id\":\"cc5c40f2-541a-49b5-8b5d-85087c2e4f37\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"425a6c09-0435-4492-8cfc-966d4bae757a\",\"type\":\"UnionRenderers\"}},\"id\":\"7f5d1caf-22e9-46cf-98e2-18cc83922816\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"bea980a3-5367-4d47-b89b-b1b030b3ef67\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"6fba7275-5eac-4da1-a13f-15afa564856d\",\"type\":\"ResetTool\"},{\"attributes\":{\"below\":[{\"id\":\"d41d1699-d865-4e1e-8e19-dfd23761e2af\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"f752cccb-d5f6-499a-8cf2-2398147088d9\",\"type\":\"LinearAxis\"}],\"plot_height\":900,\"plot_width\":900,\"renderers\":[{\"id\":\"d41d1699-d865-4e1e-8e19-dfd23761e2af\",\"type\":\"LinearAxis\"},{\"id\":\"a52a964b-2ce0-4dc7-a057-0921847315bb\",\"type\":\"Grid\"},{\"id\":\"f752cccb-d5f6-499a-8cf2-2398147088d9\",\"type\":\"LinearAxis\"},{\"id\":\"3c4f000e-d6ea-4671-b6c0-d44392ce45c0\",\"type\":\"Grid\"},{\"id\":\"bea980a3-5367-4d47-b89b-b1b030b3ef67\",\"type\":\"BoxAnnotation\"},{\"id\":\"4f2f7efb-97ab-4247-a5cf-fbdc1d8545d2\",\"type\":\"Legend\"},{\"id\":\"0dc880cc-d914-4824-a132-559022d97d50\",\"type\":\"GlyphRenderer\"},{\"id\":\"e80ea6d2-d209-4778-8065-07895a77723f\",\"type\":\"LabelSet\"}],\"title\":{\"id\":\"afe4b045-9023-4b33-974d-e93bf369255a\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"93d29d4a-eeeb-4e5e-927a-389ad9028a71\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"162e85da-9d9f-40dc-ace8-78c569ce7a95\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"84d10202-c867-41ee-89cd-0c4d0fcfe859\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"a2fb078b-164f-4a84-a1c8-550afdff24e7\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"ce2d32b8-0a1d-48b8-ac22-94d21c35f2ea\",\"type\":\"LinearScale\"}},\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null},\"id\":\"a2fb078b-164f-4a84-a1c8-550afdff24e7\",\"type\":\"DataRange1d\"},{\"attributes\":{\"level\":\"glyph\",\"plot\":{\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"source\":{\"id\":\"7f5d1caf-22e9-46cf-98e2-18cc83922816\",\"type\":\"ColumnDataSource\"},\"text\":{\"field\":\"labels\"},\"text_font\":\"CuneiformComposite\",\"text_font_size\":{\"value\":\"12pt\"},\"x\":{\"field\":\"x\"},\"x_offset\":{\"value\":5},\"y\":{\"field\":\"y\"},\"y_offset\":{\"value\":5}},\"id\":\"e80ea6d2-d209-4778-8065-07895a77723f\",\"type\":\"LabelSet\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"b213f408-847b-406c-a584-53a38f4043d5\",\"type\":\"PanTool\"},{\"id\":\"53e5ce1b-0e15-4923-8912-e53a64c8578f\",\"type\":\"WheelZoomTool\"},{\"id\":\"ca1b6de6-d3e9-41c1-ba92-e2d96c1a5cd8\",\"type\":\"BoxZoomTool\"},{\"id\":\"c9f5313d-c483-434c-b232-8c912b573ef9\",\"type\":\"SaveTool\"},{\"id\":\"6fba7275-5eac-4da1-a13f-15afa564856d\",\"type\":\"ResetTool\"},{\"id\":\"999ee0c8-9ccd-4e63-8349-0f434637b6b6\",\"type\":\"HelpTool\"}]},\"id\":\"93d29d4a-eeeb-4e5e-927a-389ad9028a71\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"d4d8cdaa-64d7-4bb7-92b8-6b87085e3f80\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"861993bf-063f-4619-94a5-ee249fc8130c\",\"type\":\"BasicTicker\"}},\"id\":\"3c4f000e-d6ea-4671-b6c0-d44392ce45c0\",\"type\":\"Grid\"},{\"attributes\":{\"overlay\":{\"id\":\"bea980a3-5367-4d47-b89b-b1b030b3ef67\",\"type\":\"BoxAnnotation\"}},\"id\":\"ca1b6de6-d3e9-41c1-ba92-e2d96c1a5cd8\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"source\":{\"id\":\"7f5d1caf-22e9-46cf-98e2-18cc83922816\",\"type\":\"ColumnDataSource\"}},\"id\":\"46548884-3ed2-402a-8db7-4c78ee9cc009\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"425a6c09-0435-4492-8cfc-966d4bae757a\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"afe4b045-9023-4b33-974d-e93bf369255a\",\"type\":\"Title\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":0.01},\"size\":{\"units\":\"screen\",\"value\":7},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"cadbc1dd-d215-42de-a829-33a796417653\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"cc5c40f2-541a-49b5-8b5d-85087c2e4f37\",\"type\":\"Selection\"},{\"attributes\":{\"items\":[{\"id\":\"963c3694-8657-4027-b38f-3c6a97085f8e\",\"type\":\"LegendItem\"}],\"plot\":{\"id\":\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"4f2f7efb-97ab-4247-a5cf-fbdc1d8545d2\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"b213f408-847b-406c-a584-53a38f4043d5\",\"type\":\"PanTool\"},{\"attributes\":{\"label\":{\"field\":\"legend\"},\"renderers\":[{\"id\":\"0dc880cc-d914-4824-a132-559022d97d50\",\"type\":\"GlyphRenderer\"}]},\"id\":\"963c3694-8657-4027-b38f-3c6a97085f8e\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"c9f5313d-c483-434c-b232-8c912b573ef9\",\"type\":\"SaveTool\"},{\"attributes\":{\"data_source\":{\"id\":\"7f5d1caf-22e9-46cf-98e2-18cc83922816\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"fbdcca76-b9d3-4897-b4cf-bd30104c0c80\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"cadbc1dd-d215-42de-a829-33a796417653\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"46548884-3ed2-402a-8db7-4c78ee9cc009\",\"type\":\"CDSView\"}},\"id\":\"0dc880cc-d914-4824-a132-559022d97d50\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"53e5ce1b-0e15-4923-8912-e53a64c8578f\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"callback\":null},\"id\":\"162e85da-9d9f-40dc-ace8-78c569ce7a95\",\"type\":\"DataRange1d\"}],\"root_ids\":[\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\"]},\"title\":\"Bokeh Application\",\"version\":\"0.13.0\"}};\n",
       "  var render_items = [{\"docid\":\"e5739e42-5f3f-4134-96e1-619c8b5f794e\",\"roots\":{\"e2fa81b0-7954-4a9a-8dba-f717a65f2563\":\"da16219a-d612-4023-9eed-a9966034b997\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "e2fa81b0-7954-4a9a-8dba-f717a65f2563"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "p2 = tsne_bokeh(model_sux, words_d, vegetables_perfumes)\n",
    "show(p2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}