Intro.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled1.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kumartr/NLP/blob/master/Intro.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "lRRCPaC96WLR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "d07b9549-e2f1-45f3-ecab-8fdf4e658acd"
      },
      "cell_type": "code",
      "source": [
        "!pip install -U nltk"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting nltk\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6f/ed/9c755d357d33bc1931e157f537721efb5b88d2c583fe593cc09603076cc3/nltk-3.4.zip (1.4MB)\n",
            "\r\u001b[K    0% |▎                               | 10kB 18.0MB/s eta 0:00:01\r\u001b[K    1% |▌                               | 20kB 3.4MB/s eta 0:00:01\r\u001b[K    2% |▊                               | 30kB 4.9MB/s eta 0:00:01\r\u001b[K    2% |█                               | 40kB 3.2MB/s eta 0:00:01\r\u001b[K    3% |█▏                              | 51kB 3.8MB/s eta 0:00:01\r\u001b[K    4% |█▍                              | 61kB 4.6MB/s eta 0:00:01\r\u001b[K    5% |█▋                              | 71kB 5.2MB/s eta 0:00:01\r\u001b[K    5% |█▉                              | 81kB 5.8MB/s eta 0:00:01\r\u001b[K    6% |██                              | 92kB 6.4MB/s eta 0:00:01\r\u001b[K    7% |██▎                             | 102kB 5.1MB/s eta 0:00:01\r\u001b[K    7% |██▌                             | 112kB 5.2MB/s eta 0:00:01\r\u001b[K    8% |██▊                             | 122kB 7.0MB/s eta 0:00:01\r\u001b[K    9% |███                             | 133kB 7.0MB/s eta 0:00:01\r\u001b[K    10% |███▏                            | 143kB 12.6MB/s eta 0:00:01\r\u001b[K    10% |███▍                            | 153kB 12.7MB/s eta 0:00:01\r\u001b[K    11% |███▋                            | 163kB 12.7MB/s eta 0:00:01\r\u001b[K    12% |████                            | 174kB 12.9MB/s eta 0:00:01\r\u001b[K    12% |████▏                           | 184kB 13.0MB/s eta 0:00:01\r\u001b[K    13% |████▍                           | 194kB 13.1MB/s eta 0:00:01\r\u001b[K    14% |████▋                           | 204kB 40.8MB/s eta 0:00:01\r\u001b[K    15% |████▉                           | 215kB 15.2MB/s eta 0:00:01\r\u001b[K    15% |█████                           | 225kB 15.1MB/s eta 0:00:01\r\u001b[K    16% |█████▎                          | 235kB 15.4MB/s eta 0:00:01\r\u001b[K    17% |█████▌                          | 245kB 15.6MB/s eta 0:00:01\r\u001b[K    17% |█████▊                          | 256kB 15.5MB/s eta 0:00:01\r\u001b[K    18% |██████                          | 266kB 14.2MB/s eta 0:00:01\r\u001b[K    19% |██████▏                         | 276kB 14.2MB/s eta 0:00:01\r\u001b[K    20% |██████▍                         | 286kB 14.0MB/s eta 0:00:01\r\u001b[K    20% |██████▋                         | 296kB 13.6MB/s eta 0:00:01\r\u001b[K    21% |██████▉                         | 307kB 13.7MB/s eta 0:00:01\r\u001b[K    22% |███████                         | 317kB 31.1MB/s eta 0:00:01\r\u001b[K    22% |███████▎                        | 327kB 15.3MB/s eta 0:00:01\r\u001b[K    23% |███████▌                        | 337kB 15.1MB/s eta 0:00:01\r\u001b[K    24% |███████▉                        | 348kB 14.3MB/s eta 0:00:01\r\u001b[K    25% |████████                        | 358kB 14.5MB/s eta 0:00:01\r\u001b[K    25% |████████▎                       | 368kB 15.8MB/s eta 0:00:01\r\u001b[K    26% |████████▌                       | 378kB 16.0MB/s eta 0:00:01\r\u001b[K    27% |████████▊                       | 389kB 16.4MB/s eta 0:00:01\r\u001b[K    27% |█████████                       | 399kB 16.6MB/s eta 0:00:01\r\u001b[K    28% |█████████▏                      | 409kB 16.9MB/s eta 0:00:01\r\u001b[K    29% |█████████▍                      | 419kB 17.4MB/s eta 0:00:01\r\u001b[K    30% |█████████▋                      | 430kB 42.1MB/s eta 0:00:01\r\u001b[K    30% |█████████▉                      | 440kB 44.4MB/s eta 0:00:01\r\u001b[K    31% |██████████                      | 450kB 47.2MB/s eta 0:00:01\r\u001b[K    32% |██████████▎                     | 460kB 45.9MB/s eta 0:00:01\r\u001b[K    32% |██████████▌                     | 471kB 45.8MB/s eta 0:00:01\r\u001b[K    33% |██████████▊                     | 481kB 45.4MB/s eta 0:00:01\r\u001b[K    34% |███████████                     | 491kB 44.4MB/s eta 0:00:01\r\u001b[K    35% |███████████▏                    | 501kB 45.9MB/s eta 0:00:01\r\u001b[K    35% |███████████▍                    | 512kB 18.2MB/s eta 0:00:01\r\u001b[K    36% |███████████▊                    | 522kB 17.8MB/s eta 0:00:01\r\u001b[K    37% |████████████                    | 532kB 17.7MB/s eta 0:00:01\r\u001b[K    37% |████████████▏                   | 542kB 17.6MB/s eta 0:00:01\r\u001b[K    38% |████████████▍                   | 552kB 18.4MB/s eta 0:00:01\r\u001b[K    39% |████████████▋                   | 563kB 18.5MB/s eta 0:00:01\r\u001b[K    40% |████████████▉                   | 573kB 18.4MB/s eta 0:00:01\r\u001b[K    40% |█████████████                   | 583kB 18.5MB/s eta 0:00:01\r\u001b[K    41% |█████████████▎                  | 593kB 18.6MB/s eta 0:00:01\r\u001b[K    42% |█████████████▌                  | 604kB 18.5MB/s eta 0:00:01\r\u001b[K    42% |█████████████▊                  | 614kB 47.9MB/s eta 0:00:01\r\u001b[K    43% |██████████████                  | 624kB 49.7MB/s eta 0:00:01\r\u001b[K    44% |██████████████▏                 | 634kB 50.9MB/s eta 0:00:01\r\u001b[K    45% |██████████████▍                 | 645kB 51.0MB/s eta 0:00:01\r\u001b[K    45% |██████████████▋                 | 655kB 50.7MB/s eta 0:00:01\r\u001b[K    46% |██████████████▉                 | 665kB 38.5MB/s eta 0:00:01\r\u001b[K    47% |███████████████                 | 675kB 39.0MB/s eta 0:00:01\r\u001b[K    47% |███████████████▎                | 686kB 39.1MB/s eta 0:00:01\r\u001b[K    48% |███████████████▋                | 696kB 39.4MB/s eta 0:00:01\r\u001b[K    49% |███████████████▉                | 706kB 39.8MB/s eta 0:00:01\r\u001b[K    50% |████████████████                | 716kB 40.5MB/s eta 0:00:01\r\u001b[K    50% |████████████████▎               | 727kB 40.8MB/s eta 0:00:01\r\u001b[K    51% |████████████████▌               | 737kB 40.5MB/s eta 0:00:01\r\u001b[K    52% |████████████████▊               | 747kB 40.5MB/s eta 0:00:01\r\u001b[K    52% |█████████████████               | 757kB 40.5MB/s eta 0:00:01\r\u001b[K    53% |█████████████████▏              | 768kB 55.8MB/s eta 0:00:01\r\u001b[K    54% |█████████████████▍              | 778kB 55.3MB/s eta 0:00:01\r\u001b[K    55% |█████████████████▋              | 788kB 51.9MB/s eta 0:00:01\r\u001b[K    55% |█████████████████▉              | 798kB 52.2MB/s eta 0:00:01\r\u001b[K    56% |██████████████████              | 808kB 52.0MB/s eta 0:00:01\r\u001b[K    57% |██████████████████▎             | 819kB 50.3MB/s eta 0:00:01\r\u001b[K    57% |██████████████████▌             | 829kB 50.7MB/s eta 0:00:01\r\u001b[K    58% |██████████████████▊             | 839kB 49.9MB/s eta 0:00:01\r\u001b[K    59% |███████████████████             | 849kB 50.7MB/s eta 0:00:01\r\u001b[K    60% |███████████████████▏            | 860kB 44.2MB/s eta 0:00:01\r\u001b[K    60% |███████████████████▌            | 870kB 44.0MB/s eta 0:00:01\r\u001b[K    61% |███████████████████▊            | 880kB 45.0MB/s eta 0:00:01\r\u001b[K    62% |████████████████████            | 890kB 47.7MB/s eta 0:00:01\r\u001b[K    62% |████████████████████▏           | 901kB 47.3MB/s eta 0:00:01\r\u001b[K    63% |████████████████████▍           | 911kB 48.0MB/s eta 0:00:01\r\u001b[K    64% |████████████████████▋           | 921kB 49.8MB/s eta 0:00:01\r\u001b[K    65% |████████████████████▉           | 931kB 48.3MB/s eta 0:00:01\r\u001b[K    65% |█████████████████████           | 942kB 47.1MB/s eta 0:00:01\r\u001b[K    66% |█████████████████████▎          | 952kB 46.8MB/s eta 0:00:01\r\u001b[K    67% |█████████████████████▌          | 962kB 53.8MB/s eta 0:00:01\r\u001b[K    67% |█████████████████████▊          | 972kB 53.8MB/s eta 0:00:01\r\u001b[K    68% |██████████████████████          | 983kB 53.1MB/s eta 0:00:01\r\u001b[K    69% |██████████████████████▏         | 993kB 51.9MB/s eta 0:00:01\r\u001b[K    70% |██████████████████████▍         | 1.0MB 51.6MB/s eta 0:00:01\r\u001b[K    70% |██████████████████████▋         | 1.0MB 50.5MB/s eta 0:00:01\r\u001b[K    71% |██████████████████████▉         | 1.0MB 49.7MB/s eta 0:00:01\r\u001b[K    72% |███████████████████████▏        | 1.0MB 51.0MB/s eta 0:00:01\r\u001b[K    72% |███████████████████████▍        | 1.0MB 53.6MB/s eta 0:00:01\r\u001b[K    73% |███████████████████████▋        | 1.1MB 53.5MB/s eta 0:00:01\r\u001b[K    74% |███████████████████████▉        | 1.1MB 54.2MB/s eta 0:00:01\r\u001b[K    75% |████████████████████████        | 1.1MB 53.0MB/s eta 0:00:01\r\u001b[K    75% |████████████████████████▎       | 1.1MB 52.5MB/s eta 0:00:01\r\u001b[K    76% |████████████████████████▌       | 1.1MB 40.5MB/s eta 0:00:01\r\u001b[K    77% |████████████████████████▊       | 1.1MB 40.5MB/s eta 0:00:01\r\u001b[K    77% |█████████████████████████       | 1.1MB 38.9MB/s eta 0:00:01\r\u001b[K    78% |█████████████████████████▏      | 1.1MB 39.5MB/s eta 0:00:01\r\u001b[K    79% |█████████████████████████▍      | 1.1MB 39.9MB/s eta 0:00:01\r\u001b[K    80% |█████████████████████████▋      | 1.1MB 40.1MB/s eta 0:00:01\r\u001b[K    80% |█████████████████████████▉      | 1.2MB 40.5MB/s eta 0:00:01\r\u001b[K    81% |██████████████████████████      | 1.2MB 40.0MB/s eta 0:00:01\r\u001b[K    82% |██████████████████████████▎     | 1.2MB 40.9MB/s eta 0:00:01\r\u001b[K    82% |██████████████████████████▌     | 1.2MB 41.5MB/s eta 0:00:01\r\u001b[K    83% |██████████████████████████▊     | 1.2MB 55.4MB/s eta 0:00:01\r\u001b[K    84% |███████████████████████████     | 1.2MB 57.1MB/s eta 0:00:01\r\u001b[K    85% |███████████████████████████▎    | 1.2MB 61.6MB/s eta 0:00:01\r\u001b[K    85% |███████████████████████████▌    | 1.2MB 60.4MB/s eta 0:00:01\r\u001b[K    86% |███████████████████████████▊    | 1.2MB 59.7MB/s eta 0:00:01\r\u001b[K    87% |████████████████████████████    | 1.2MB 59.3MB/s eta 0:00:01\r\u001b[K    87% |████████████████████████████▏   | 1.3MB 59.3MB/s eta 0:00:01\r\u001b[K    88% |████████████████████████████▍   | 1.3MB 60.1MB/s eta 0:00:01\r\u001b[K    89% |████████████████████████████▋   | 1.3MB 59.0MB/s eta 0:00:01\r\u001b[K    90% |████████████████████████████▉   | 1.3MB 57.4MB/s eta 0:00:01\r\u001b[K    90% |█████████████████████████████   | 1.3MB 56.9MB/s eta 0:00:01\r\u001b[K    91% |█████████████████████████████▎  | 1.3MB 56.2MB/s eta 0:00:01\r\u001b[K    92% |█████████████████████████████▌  | 1.3MB 56.0MB/s eta 0:00:01\r\u001b[K    92% |█████████████████████████████▊  | 1.3MB 55.4MB/s eta 0:00:01\r\u001b[K    93% |██████████████████████████████  | 1.3MB 55.3MB/s eta 0:00:01\r\u001b[K    94% |██████████████████████████████▏ | 1.4MB 54.0MB/s eta 0:00:01\r\u001b[K    95% |██████████████████████████████▍ | 1.4MB 52.9MB/s eta 0:00:01\r\u001b[K    95% |██████████████████████████████▋ | 1.4MB 53.2MB/s eta 0:00:01\r\u001b[K    96% |███████████████████████████████ | 1.4MB 53.4MB/s eta 0:00:01\r\u001b[K    97% |███████████████████████████████▏| 1.4MB 53.2MB/s eta 0:00:01\r\u001b[K    97% |███████████████████████████████▍| 1.4MB 31.8MB/s eta 0:00:01\r\u001b[K    98% |███████████████████████████████▋| 1.4MB 31.3MB/s eta 0:00:01\r\u001b[K    99% |███████████████████████████████▉| 1.4MB 31.5MB/s eta 0:00:01\r\u001b[K    100% |████████████████████████████████| 1.4MB 11.7MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from nltk) (1.11.0)\n",
            "Collecting singledispatch (from nltk)\n",
            "  Downloading https://files.pythonhosted.org/packages/c5/10/369f50bcd4621b263927b0a1519987a04383d4a98fb10438042ad410cf88/singledispatch-3.4.0.3-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: nltk\n",
            "  Running setup.py bdist_wheel for nltk ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/4b/c8/24/b2343664bcceb7147efeb21c0b23703a05b23fcfeaceaa2a1e\n",
            "Successfully built nltk\n",
            "Installing collected packages: singledispatch, nltk\n",
            "  Found existing installation: nltk 3.2.5\n",
            "    Uninstalling nltk-3.2.5:\n",
            "      Successfully uninstalled nltk-3.2.5\n",
            "Successfully installed nltk-3.4 singledispatch-3.4.0.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Y0Ih5-Z_6ZlN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Sample code to remove noisy words from a text\n",
        "\n",
        "noise_list = [\"is\", \"a\", \"this\", \"...\"] \n",
        "def _remove_noise(input_text):\n",
        "    words = input_text.split() \n",
        "    noise_free_words = [word for word in words if word not in noise_list] \n",
        "    noise_free_text = \" \".join(noise_free_words) \n",
        "    return noise_free_text\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vDAfhOmN7GP5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "84e187c3-14c6-4f8d-e13c-4416349ff7de"
      },
      "cell_type": "code",
      "source": [
        "_remove_noise(\"this is a sample text\")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'sample text'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "metadata": {
        "id": "tU1-Pman7aWs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "096819b2-8e82-4310-e64f-e793bbc76aed"
      },
      "cell_type": "code",
      "source": [
        "# Sample code to remove a regex pattern \n",
        "import re \n",
        "\n",
        "def _remove_regex(input_text, regex_pattern):\n",
        "    urls = re.finditer(regex_pattern, input_text) \n",
        "    for i in urls: \n",
        "        input_text = re.sub(i.group().strip(), '', input_text)\n",
        "    return input_text\n",
        "\n",
        "regex_pattern = \"#[\\w]*\"  \n",
        "\n",
        "_remove_regex(\"remove this #hashtag from analytics vidhya\", regex_pattern)\n"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'remove this  from analytics vidhya'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "L5SKtsPm8WM8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "8641d789-6b31-4c9b-cb86-5a86844dfc86"
      },
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "\n",
        "from nltk.stem.wordnet import WordNetLemmatizer \n",
        "lem = WordNetLemmatizer()\n",
        "\n",
        "from nltk.stem.porter import PorterStemmer \n",
        "stem = PorterStemmer()\n",
        "\n",
        "word = \"multiplying\" \n",
        "lem.lemmatize(word, \"v\")\n",
        "stem.stem(word)\n"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'multipli'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "metadata": {
        "id": "8m08_jGg9UcZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "7f9ea32d-f81c-4394-e185-d54279794179"
      },
      "cell_type": "code",
      "source": [
        "lem.lemmatize(word, \"v\")"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'multiply'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "qBMEm-eE9VMC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "f998ba07-b69c-482f-90a2-3790cf7d7747"
      },
      "cell_type": "code",
      "source": [
        "lookup_dict = {'rt':'Retweet', 'dm':'direct message', \"awsm\" : \"awesome\", \"luv\" :\"love\"}\n",
        "def _lookup_words(input_text):\n",
        "    words = input_text.split() \n",
        "    new_words = [] \n",
        "    for word in words:\n",
        "        if word.lower() in lookup_dict:\n",
        "            word = lookup_dict[word.lower()]\n",
        "        new_words.append(word) \n",
        "    new_text = \" \".join(new_words) \n",
        "    return new_text\n",
        "\n",
        "_lookup_words(\"RT this is a retweeted tweet by Shivam Bansal\")"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Retweet this is a retweeted tweet by Shivam Bansal'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "lcUobEytYj7_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 149
        },
        "outputId": "c2f54372-acd2-4b3d-e8d4-b4d5d9b3b328"
      },
      "cell_type": "code",
      "source": [
        "from nltk import word_tokenize, pos_tag\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "text = \"I am learning Natural Language Processing on Analytics Vidhya\"\n",
        "tokens = word_tokenize(text)\n",
        "print(pos_tag(tokens))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('I', 'PRP'), ('am', 'VBP'), ('learning', 'VBG'), ('Natural', 'NNP'), ('Language', 'NNP'), ('Processing', 'NNP'), ('on', 'IN'), ('Analytics', 'NNP'), ('Vidhya', 'NNP')]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "ucfjvc2WbIsY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 433
        },
        "outputId": "e517831a-7e67-4caf-affb-b7265e2af1e4"
      },
      "cell_type": "code",
      "source": [
        "!pip install --upgrade gensim\n",
        "!pip install --upgrade corpora\n",
        "!pip install --upgrade corpus\n",
        "\n",
        "\n",
        "doc1 = \"Sugar is bad to consume. My sister likes to have sugar, but not my father.\" \n",
        "doc2 = \"My father spends a lot of time driving my sister around to dance practice.\"\n",
        "doc3 = \"Doctors suggest that driving may cause increased stress and blood pressure.\"\n",
        "doc_complete = [doc1, doc2, doc3]\n",
        "doc_clean = [doc.split() for doc in doc_complete]\n",
        "\n",
        "import gensim\n",
        "import gensim.corpora\n",
        "\n",
        "\n",
        "# Creating the term dictionary of our corpus, where every unique term is assigned an index.  \n",
        "dictionary = gensim.corpora.Dictionary(doc_clean)\n",
        "\n",
        "# Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above. \n",
        "doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean]\n",
        "\n",
        "# Creating the object for LDA model using gensim library\n",
        "Lda = gensim.models.ldamodel.LdaModel\n",
        "\n",
        "# Running and Training LDA model on the document term matrix\n",
        "ldamodel = Lda(doc_term_matrix, num_topics=3, id2word = dictionary, passes=50)\n",
        "\n",
        "# Results \n",
        "print(ldamodel.print_topics())"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: gensim in /usr/local/lib/python3.6/dist-packages (3.6.0)\n",
            "Requirement already satisfied, skipping upgrade: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.11.0)\n",
            "Requirement already satisfied, skipping upgrade: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.7.1)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.14.6)\n",
            "Requirement already satisfied, skipping upgrade: bz2file in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (0.98)\n",
            "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (1.9.57)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.18.4)\n",
            "Requirement already satisfied, skipping upgrade: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.49.0)\n",
            "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.9.3)\n",
            "Requirement already satisfied, skipping upgrade: s3transfer<0.2.0,>=0.1.10 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.1.13)\n",
            "Requirement already satisfied, skipping upgrade: botocore<1.13.0,>=1.12.57 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (1.12.57)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2018.10.15)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.22)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.6)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.57->boto3->smart-open>=1.2.1->gensim) (0.14)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.57->boto3->smart-open>=1.2.1->gensim) (2.5.3)\n",
            "Requirement already up-to-date: corpora in /usr/local/lib/python3.6/dist-packages (1.0)\n",
            "Requirement already up-to-date: corpus in /usr/local/lib/python3.6/dist-packages (0.4.2)\n",
            "[(0, '0.089*\"to\" + 0.051*\"My\" + 0.051*\"sister\" + 0.051*\"my\" + 0.051*\"likes\" + 0.051*\"sugar,\" + 0.051*\"not\" + 0.051*\"Sugar\" + 0.051*\"but\" + 0.051*\"have\"'), (1, '0.053*\"driving\" + 0.053*\"my\" + 0.053*\"My\" + 0.053*\"sister\" + 0.053*\"father\" + 0.053*\"around\" + 0.053*\"a\" + 0.053*\"time\" + 0.053*\"spends\" + 0.053*\"lot\"'), (2, '0.060*\"driving\" + 0.060*\"increased\" + 0.060*\"and\" + 0.060*\"blood\" + 0.060*\"cause\" + 0.060*\"may\" + 0.060*\"suggest\" + 0.060*\"stress\" + 0.060*\"Doctors\" + 0.060*\"pressure.\"')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9ZapaXWYim__",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "ba8c5427-bb84-403b-88a5-58383e524512"
      },
      "cell_type": "code",
      "source": [
        "def generate_ngrams(text, n):\n",
        "    words = text.split()\n",
        "    output = []  \n",
        "    for i in range(len(words)-n+1):\n",
        "        output.append(words[i:i+n])\n",
        "    return output\n",
        "\n",
        "generate_ngrams('this is a sample text', 2)\n"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['this', 'is'], ['is', 'a'], ['a', 'sample'], ['sample', 'text']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "metadata": {
        "id": "MIGV67HiioFe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "outputId": "f9e56fdc-caa1-4add-b8fb-f24c646029b4"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "obj = TfidfVectorizer()\n",
        "corpus = ['This is sample document.', 'another random document.', 'third sample document text']\n",
        "X = obj.fit_transform(corpus)\n",
        "print (X)"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  (0, 7)\t0.5844829010200651\n",
            "  (0, 2)\t0.5844829010200651\n",
            "  (0, 4)\t0.444514311537431\n",
            "  (0, 1)\t0.34520501686496574\n",
            "  (1, 1)\t0.3853716274664007\n",
            "  (1, 0)\t0.652490884512534\n",
            "  (1, 3)\t0.652490884512534\n",
            "  (2, 4)\t0.444514311537431\n",
            "  (2, 1)\t0.34520501686496574\n",
            "  (2, 6)\t0.5844829010200651\n",
            "  (2, 5)\t0.5844829010200651\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GKHlVKsrtGOw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "outputId": "6ace39a6-02a2-4637-d613-0a117ef9d29f"
      },
      "cell_type": "code",
      "source": [
        "from gensim.models import Word2Vec\n",
        "sentences = [['data', 'science'], ['vidhya', 'science', 'data', 'analytics'],['machine', 'learning'], ['deep', 'learning']]\n",
        "model = Word2Vec(sentences, min_count = 1)\n",
        "print (model.similarity('data', 'science'))\n",
        "\n",
        "print (model['learning'])"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-0.035537764\n",
            "[ 1.6674837e-03  3.8019589e-03 -4.4530001e-03 -4.1990046e-04\n",
            " -1.0664921e-03 -1.9685132e-03 -4.0113986e-03  4.9367710e-04\n",
            " -4.8962808e-03 -4.2713401e-03  4.8858440e-03 -2.5366365e-03\n",
            " -4.6215653e-03  3.6981432e-03  1.6131763e-04  3.3377334e-03\n",
            " -2.2189922e-03  7.6688809e-04  5.0042768e-04 -1.5961276e-03\n",
            " -2.8059806e-04  2.3710413e-03  3.2811484e-03  8.1652665e-04\n",
            " -2.3568647e-03 -2.9333194e-03  2.8791903e-03  6.3822529e-04\n",
            "  4.4684717e-04  1.9522732e-03  8.3009509e-04  6.9440273e-04\n",
            " -2.2804942e-03 -4.9088555e-03  2.3034695e-03 -4.4662510e-03\n",
            " -1.8379214e-03 -3.1208058e-03  3.0260491e-03 -2.2757235e-03\n",
            "  2.9647048e-03 -1.4055354e-03 -4.6273661e-03 -4.8188018e-03\n",
            " -1.6892443e-03 -3.0722085e-03  4.7295238e-03  4.5917286e-03\n",
            "  3.6563685e-03 -2.4883570e-03  3.4532670e-04  4.1071409e-03\n",
            "  5.1735848e-04 -1.5334516e-03 -2.4158091e-03 -1.9203996e-03\n",
            " -4.5427848e-03  1.0680027e-03  3.9046451e-03  1.4577399e-03\n",
            " -4.8456229e-03  2.8765677e-03  9.5568673e-04 -4.1900962e-04\n",
            " -1.4544878e-03  1.3683642e-03  2.3312494e-03 -5.4618555e-05\n",
            " -4.4738830e-04 -1.9489021e-04 -2.7674076e-03  4.5897644e-03\n",
            " -3.2291787e-03 -1.2214208e-03 -2.5026007e-03 -2.3467117e-03\n",
            " -1.8916928e-03  2.1261033e-03  2.3600725e-03 -3.1530419e-03\n",
            "  1.1474374e-04 -1.8317589e-04  4.4654221e-03 -4.0492546e-03\n",
            "  8.7231398e-04 -5.2708376e-04  9.4407116e-04 -8.7262233e-05\n",
            "  1.1093179e-03  2.4498838e-03 -2.6695451e-03 -4.7963629e-03\n",
            " -2.3844035e-03 -4.3749753e-03  1.2733687e-03  5.3317950e-04\n",
            "  1.0279275e-03  4.3354174e-03 -2.8919007e-03 -5.1082153e-04]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n",
            "  after removing the cwd from sys.path.\n",
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "lOwsK4Aitrnt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 149
        },
        "outputId": "526ac55e-9409-4e9e-c84d-22e8d87d3dfe"
      },
      "cell_type": "code",
      "source": [
        "!pip install textblob\n",
        "import textblob\n",
        "from textblob.classifiers import NaiveBayesClassifier as NBC\n",
        "from textblob import TextBlob\n",
        "training_corpus = [\n",
        "                   ('I am exhausted of this work.', 'Class_B'),\n",
        "                   (\"I can't cooperate with this\", 'Class_B'),\n",
        "                   ('He is my badest enemy!', 'Class_B'),\n",
        "                   ('My management is poor.', 'Class_B'),\n",
        "                   ('I love this burger.', 'Class_A'),\n",
        "                   ('This is an brilliant place!', 'Class_A'),\n",
        "                   ('I feel very good about these dates.', 'Class_A'),\n",
        "                   ('This is my best work.', 'Class_A'),\n",
        "                   (\"What an awesome view\", 'Class_A'),\n",
        "                   ('I do not like this dish', 'Class_B')]\n",
        "test_corpus = [\n",
        "                (\"I am not feeling well today.\", 'Class_B'), \n",
        "                (\"I feel brilliant!\", 'Class_A'), \n",
        "                ('Gary is a friend of mine.', 'Class_A'), \n",
        "                (\"I can't believe I'm doing this.\", 'Class_B'), \n",
        "                ('The date was good.', 'Class_A'), ('I do not enjoy my job', 'Class_B')]\n",
        "\n",
        "model = NBC(training_corpus) \n",
        "print(model.classify(\"Their codes are amazing.\"))\n",
        "print(model.classify(\"I don't like their computer.\"))\n",
        "print(model.accuracy(test_corpus))"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: textblob in /usr/local/lib/python3.6/dist-packages (0.15.2)\n",
            "Requirement already satisfied: nltk>=3.1 in /usr/local/lib/python3.6/dist-packages (from textblob) (3.4)\n",
            "Requirement already satisfied: singledispatch in /usr/local/lib/python3.6/dist-packages (from nltk>=3.1->textblob) (3.4.0.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk>=3.1->textblob) (1.11.0)\n",
            "Class_A\n",
            "Class_B\n",
            "0.8333333333333334\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "X9fosGkf4H3T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 149
        },
        "outputId": "179e80dd-acea-48e0-cef4-2750f87464c4"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer \n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn import svm \n",
        "\n",
        "# preparing data for SVM model (using the same training_corpus, test_corpus from naive bayes example)\n",
        "train_data = []\n",
        "train_labels = []\n",
        "for row in training_corpus:\n",
        "    train_data.append(row[0])\n",
        "    train_labels.append(row[1])\n",
        "\n",
        "test_data = [] \n",
        "test_labels = [] \n",
        "for row in test_corpus:\n",
        "    test_data.append(row[0]) \n",
        "    test_labels.append(row[1])\n",
        "\n",
        "# Create feature vectors \n",
        "vectorizer = TfidfVectorizer(min_df=4, max_df=0.9)\n",
        "# Train the feature vectors\n",
        "train_vectors = vectorizer.fit_transform(train_data)\n",
        "# Apply model on test data \n",
        "test_vectors = vectorizer.transform(test_data)\n",
        "\n",
        "# Perform classification with SVM, kernel=linear \n",
        "model = svm.SVC(kernel='linear') \n",
        "model.fit(train_vectors, train_labels) \n",
        "prediction = model.predict(test_vectors)\n",
        "prediction\n",
        "print (classification_report(test_labels, prediction))"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "             precision    recall  f1-score   support\n",
            "\n",
            "    Class_A       0.50      0.67      0.57         3\n",
            "    Class_B       0.50      0.33      0.40         3\n",
            "\n",
            "avg / total       0.50      0.50      0.49         6\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "97aRSH65416S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "4.2 Text Matching / Similarity"
      ]
    },
    {
      "metadata": {
        "id": "q33M-Wuc44i_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "071fa1af-c1ed-4df4-84a4-cf547e98791a"
      },
      "cell_type": "code",
      "source": [
        "def levenshtein(s1,s2): \n",
        "    if len(s1) > len(s2):\n",
        "        s1,s2 = s2,s1 \n",
        "    distances = range(len(s1) + 1) \n",
        "    for index2,char2 in enumerate(s2):\n",
        "        newDistances = [index2+1]\n",
        "        for index1,char1 in enumerate(s1):\n",
        "            if char1 == char2:\n",
        "                newDistances.append(distances[index1]) \n",
        "            else:\n",
        "                 newDistances.append(1 + min((distances[index1], distances[index1+1], newDistances[-1]))) \n",
        "        distances = newDistances \n",
        "    return distances[-1]\n",
        "\n",
        "print(levenshtein(\"analyze\",\"analyse\"))"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "zCy3mGQg5HBt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        },
        "outputId": "a297fe4a-fda2-4f9f-e21b-80c9f75a696c"
      },
      "cell_type": "code",
      "source": [
        "!pip install fuzzy\n",
        "import fuzzy \n",
        "soundex = fuzzy.Soundex(4) \n",
        "print(soundex('ankit'))"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: fuzzy in /usr/local/lib/python3.6/dist-packages (1.2.2)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "UnicodeDecodeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-77-4f327fdde5d6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfuzzy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msoundex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfuzzy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSoundex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msoundex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ankit'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32msrc/fuzzy.pyx\u001b[0m in \u001b[0;36mfuzzy.Soundex.__call__\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'ascii' codec can't decode byte 0xe8 in position 0: ordinal not in range(128)"
          ]
        }
      ]
    }
  ]
}