TensorFlow1_Basics_of_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TensorFlow1_Basics-of-NLP.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPNqo0cKiusUXZ8xIbnZzP8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/utkarsh0702/Tensorflow/blob/master/TensorFlow1_Basics_of_NLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M3A1sh-52_Ey",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L7eso6US4h_-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "87bcf51c-6945-4f0d-f2a0-a4357403eb53"
      },
      "source": [
        "print(tf.__version__)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0-rc2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hwSblA154nA9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "20f68120-b70d-4b99-c025-8ac99125c4ea"
      },
      "source": [
        "#Key value pair using keras in nlp\n",
        "sentences=[\n",
        "           'I love my dog',\n",
        "           'I love my cat',\n",
        "           'You love my dog!'\n",
        "]\n",
        "token= Tokenizer(num_words=100)\n",
        "token.fit_on_texts(sentences)\n",
        "word_index= token.word_index\n",
        "print(word_index)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'love': 1, 'my': 2, 'i': 3, 'dog': 4, 'cat': 5, 'you': 6}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kyc1DsC25Z7N",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "7e812963-8528-4329-e3a5-92a5a5acc390"
      },
      "source": [
        "# Sequencing the key value pair\n",
        "sentences=[\n",
        "           'I love my dog',\n",
        "           'I love my cat',\n",
        "           'You love my dog!',\n",
        "           'Do you think my dog is amazing?'\n",
        "]\n",
        "token= Tokenizer(num_words=100)\n",
        "token.fit_on_texts(sentences)\n",
        "word_index= token.word_index\n",
        "sequence= token.texts_to_sequences(sentences)\n",
        "print(word_index)\n",
        "print(sequence)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'my': 1, 'love': 2, 'dog': 3, 'i': 4, 'you': 5, 'cat': 6, 'do': 7, 'think': 8, 'is': 9, 'amazing': 10}\n",
            "[[4, 2, 1, 3], [4, 2, 1, 6], [5, 2, 1, 3], [7, 5, 8, 1, 3, 9, 10]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ODGCKwrz531o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "e669eee8-ee20-48da-f1b7-ac3e486030c9"
      },
      "source": [
        "#for the unseen data to avoid conflicts and loss\n",
        "sentences=[\n",
        "           'I love my dog',\n",
        "           'I love my cat',\n",
        "           'You love my dog!',\n",
        "           'Do you think my dog is amazing?'\n",
        "]\n",
        "test_sentences=[\n",
        "                 'I want my cycle',\n",
        "                 'Dog is my favorite animal'\n",
        "]\n",
        "token= Tokenizer(num_words=100, oov_token='<OOV>')\n",
        "token.fit_on_texts(sentences)\n",
        "word_index= token.word_index\n",
        "sequence= token.texts_to_sequences(sentences)\n",
        "# for testing on unseen words and sentences\n",
        "test_sequence= token.texts_to_sequences(test_sentences)\n",
        "print(word_index)\n",
        "print('--------------------Train Data----------------------')\n",
        "print(sequence)\n",
        "print('---------------------Test Data----------------------')\n",
        "print(test_sequence)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'<OOV>': 1, 'my': 2, 'love': 3, 'dog': 4, 'i': 5, 'you': 6, 'cat': 7, 'do': 8, 'think': 9, 'is': 10, 'amazing': 11}\n",
            "--------------------Train Data----------------------\n",
            "[[5, 3, 2, 4], [5, 3, 2, 7], [6, 3, 2, 4], [8, 6, 9, 2, 4, 10, 11]]\n",
            "---------------------Test Data----------------------\n",
            "[[5, 1, 2, 1], [4, 10, 2, 1, 1]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4bIPuig57Eu6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "e7af5851-ff08-43fa-dfb6-033292a71dd6"
      },
      "source": [
        "#Padding to make word sequence of each letter of equal length\n",
        "sentences=[\n",
        "           'I love my dog',\n",
        "           'I love my cat',\n",
        "           'You love my dog!',\n",
        "           'Do you think my dog is amazing?'\n",
        "]\n",
        "\n",
        "token= Tokenizer(num_words=100, oov_token='<OOV>')\n",
        "token.fit_on_texts(sentences)\n",
        "word_index= token.word_index\n",
        "sequence= token.texts_to_sequences(sentences)\n",
        "#padding\n",
        "padded= pad_sequences(sequence)\n",
        "#padding from the end\n",
        "padded1= pad_sequences(sequence, padding='post')\n",
        "#padding with maximum length\n",
        "padded2= pad_sequences(sequence, maxlen=5)\n",
        "print(word_index)\n",
        "print(sequence)\n",
        "print('---------------Padding------------------')\n",
        "print(padded)\n",
        "print('---------------Padding from the end------------')\n",
        "print(padded1)\n",
        "print('---------------Padding with max length------------')\n",
        "print(padded2)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'<OOV>': 1, 'my': 2, 'love': 3, 'dog': 4, 'i': 5, 'you': 6, 'cat': 7, 'do': 8, 'think': 9, 'is': 10, 'amazing': 11}\n",
            "[[5, 3, 2, 4], [5, 3, 2, 7], [6, 3, 2, 4], [8, 6, 9, 2, 4, 10, 11]]\n",
            "---------------Padding------------------\n",
            "[[ 0  0  0  5  3  2  4]\n",
            " [ 0  0  0  5  3  2  7]\n",
            " [ 0  0  0  6  3  2  4]\n",
            " [ 8  6  9  2  4 10 11]]\n",
            "---------------Padding from the end------------\n",
            "[[ 5  3  2  4  0  0  0]\n",
            " [ 5  3  2  7  0  0  0]\n",
            " [ 6  3  2  4  0  0  0]\n",
            " [ 8  6  9  2  4 10 11]]\n",
            "---------------Padding with max length------------\n",
            "[[ 0  5  3  2  4]\n",
            " [ 0  5  3  2  7]\n",
            " [ 0  6  3  2  4]\n",
            " [ 9  2  4 10 11]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PLEpDgaL8nya",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}