spacy_tokenizer_changing_default_infixes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Purpose of this notebook is to attempt to modify rules in the default spacy tokenizer to fix common issues in clinical text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spacy version: 2.0.12\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "import re\n",
    "from spacy.tokenizer import Tokenizer\n",
    "\n",
    "print('Spacy version: {}'.format(spacy.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_text = r'Pt c\\o heart#burn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's try the default tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens in default tokenizer\n",
      "Pt\n",
      "c\\o\n",
      "heart#burn\n"
     ]
    }
   ],
   "source": [
    "default_doc = default_nlp(example_text)\n",
    "\n",
    "print('Tokens in default tokenizer')\n",
    "for token in default_doc:\n",
    "    print(token.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE: This was originally found by looking at GitHub issues on spacy:\n",
    "# https://github.com/explosion/spaCy/issues/1494\n",
    "\n",
    "# now let's try to modify the infix behavior\n",
    "def my_tokenizer(nlp):\n",
    "    \n",
    "    infix_re = re.compile(r'''[^a-z0-9]''')\n",
    "    \n",
    "    return Tokenizer(nlp.vocab,\n",
    "                     {},\n",
    "                     infix_finditer=infix_re.finditer\n",
    "    )\n",
    "\n",
    "custom_nlp = spacy.load('en')\n",
    "custom_nlp.tokenizer = my_tokenizer(custom_nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens in default tokenizer\n",
      "Pt\n",
      "c\n",
      "\\\n",
      "o\n",
      "heart\n",
      "#\n",
      "burn\n"
     ]
    }
   ],
   "source": [
    "custom_doc = custom_nlp(example_text)\n",
    "\n",
    "print('Tokens in default tokenizer')\n",
    "for token in custom_doc:\n",
    "    print(token.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:spacy]",
   "language": "python",
   "name": "conda-env-spacy-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}