main.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l17WznaGAB5s",
        "outputId": "9f793677-9d7e-4a04-9e61-88c7a9c0c6af"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8tpW5nkMc53m",
        "outputId": "7eefa875-1402-4f86-e2ee-3385791d366b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m281.3/281.3 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m453.8/453.8 kB\u001b[0m \u001b[31m31.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.7/199.7 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting Levenshtein\n",
            "  Downloading Levenshtein-0.23.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (169 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m169.4/169.4 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rapidfuzz<4.0.0,>=3.1.0 (from Levenshtein)\n",
            "  Downloading rapidfuzz-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: rapidfuzz, Levenshtein\n",
            "Successfully installed Levenshtein-0.23.0 rapidfuzz-3.5.2\n"
          ]
        }
      ],
      "source": [
        "# Install PySpark and Spark NLP\n",
        "!pip install -q pyspark==3.3.0 spark-nlp==4.2.8\n",
        "!pip install Levenshtein"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "lIGUfEtSDbEi"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import sparknlp\n",
        "import pyspark.sql.functions as F\n",
        "\n",
        "from pyspark.ml import Pipeline\n",
        "from pyspark.sql import SparkSession\n",
        "from sparknlp.annotator import *\n",
        "from sparknlp.base import *\n",
        "from sparknlp.pretrained import PretrainedPipeline\n",
        "from pyspark.sql.types import StringType, IntegerType\n",
        "from transformers import pipeline\n",
        "import Levenshtein"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "cbNneAVCLU1y"
      },
      "outputs": [],
      "source": [
        "spark = sparknlp.start()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gvn2hTw6DbIH",
        "outputId": "110bf1fc-135b-4b9c-cebe-e142483152ae"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "bert_token_classifier_hi_en_ner download started this may take some time.\n",
            "Approximate size to download 634.9 MB\n",
            "[OK!]\n"
          ]
        }
      ],
      "source": [
        "document_assembler = DocumentAssembler()\\\n",
        "        .setInputCol('text')\\\n",
        "        .setOutputCol('document')\n",
        "\n",
        "sentence_detector = SentenceDetector() \\\n",
        "        .setInputCols(['document'])\\\n",
        "        .setOutputCol('sentence')\n",
        "\n",
        "tokenizer = Tokenizer()\\\n",
        "        .setInputCols(['sentence']) \\\n",
        "        .setOutputCol('token')\n",
        "\n",
        "tokenClassifier_loaded = BertForTokenClassification.pretrained(\"bert_token_classifier_hi_en_ner\",\"hi\")\\\n",
        "        .setInputCols([\"sentence\",'token'])\\\n",
        "        .setOutputCol(\"ner\")\n",
        "\n",
        "ner_converter = NerConverter()\\\n",
        "        .setInputCols([\"sentence\",\"token\",\"ner\"])\\\n",
        "        .setOutputCol(\"ner_chunk\")\n",
        "\n",
        "nlp_pipeline = Pipeline(stages=[document_assembler,\n",
        "                                sentence_detector,\n",
        "                                tokenizer,\n",
        "                                tokenClassifier_loaded,\n",
        "                                ner_converter])\n",
        "\n",
        "checkpoint = \"/content/drive/MyDrive/NER/checkpoint-3135\"\n",
        "token_classifier = pipeline(\n",
        "    \"token-classification\", model=checkpoint, aggregation_strategy=\"simple\"\n",
        ")\n",
        "\n",
        "# text_list =[\"\"\"वॉरेन एडवर्ड बफेट (Warren Buffet) (अगस्त 30 (August 30), 1930 को ओमाहा (Omaha), नेब्रास्का (Nebraska) में पैदा हुए) एक अमेरिकी निवेशक (investor), व्यवसायी और परोपकारी (philanthropist) व्यक्तित्व हैं।\"\"\"]\n",
        "\n",
        "# df = spark.createDataFrame(text_list, StringType()).toDF(\"text\")\n",
        "# result = nlp_pipeline.fit(df).transform(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Swe8hQKTokWW"
      },
      "outputs": [],
      "source": [
        "# text_list =[\"Are Jaipur and Ajmer the same state?\"]\n",
        "# text_list = [\"AMong jaipur, Ajmer and Baroda whih of these have a higher population than Surat\"]\n",
        "# text_list = [\"Which has higher average temperature in June, AHmedabad or Gandhinagar\"]\n",
        "# text_list = [\"Where is zebara? is it in amdavad?\"]\n",
        "# text_list = [\"The new england journal of medicine is the best medical journal in the world\"]\n",
        "# text_list = [\"Can I visit new york, mars, sun and delhi on the same day?\"]\n",
        "# text_list = [\"Can I visit new york mars sun and delhi on the same day?\"]\n",
        "# text_list = [\"where can i find lakes near Ahmedabad\"]\n",
        "# text_list = [\"where can I find lakes near Amdavad\"]\n",
        "# text_list = [\"is there water in delli?\"]\n",
        "# text_list = [\"Is it a good time to visit Prince edward island?\"]\n",
        "# text_list = [\"which has higher average temperature in june, ahmedabad or gandhinagar\"]\n",
        "# text_list = [\"Which Has Higher Average Temperature In June, Ahmdvad Or Gandhingr\"]\n",
        "# text_list = [\"temperature at amdavad is high\"]\n",
        "# text_list = [\"The zoo is located in Abc\"]\n",
        "# text_list = [\"The zoo is located in Abc\"]\n",
        "# text_list = [\"अहमदाबाद का तापमान मध्य प्रदेश से भी ज्यादा है\"]\n",
        "# text_list = [\"अमदाबद का तापमान मध्यदेश से भी ज्यादा है\"]\n",
        "# text_list = [\"Name of my daughter is India\"]\n",
        "# text_list = [\"I hate Gujarat but i love faafada\"]\n",
        "# text_list = [\"Anjeer is my favourite\"]\n",
        "# text_list = [\"Farah Went To Kushk\"]\n",
        "# text_list = [\"Temperature At Abc Is Higher Than Temperature at xyz\"]\n",
        "# text_list = [\"Venus has a travel planned to mars\"]\n",
        "# text_list = [\"The dal lake is in sri nagar\"]\n",
        "# text_list = [\"What happened in Tamil Nadu\"]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "jkIYifAmJMYS"
      },
      "outputs": [],
      "source": [
        "import codecs,string\n",
        "def is_hindi(character):\n",
        "    maxchar = max(character)\n",
        "    if u'\\u0900' <= maxchar <= u'\\u097f':\n",
        "        return True\n",
        "    else:\n",
        "      return False\n",
        "\n",
        "def findword(text, s, e):\n",
        "  while s>0 and text[s-1] != ' ' and text[s-1] != ',' and text[s-1] != '.' and text[s-1] != '?':\n",
        "    s-=1\n",
        "  while e<len(text) and text[e] != ' ' and text[e] != ',' and text[e] != '.' and text[e] != '?':\n",
        "    e+=1\n",
        "  return text[s:e]\n",
        "\n",
        "def combinedOutput(extracted_list, output, text_list):\n",
        "  namelist1 = {}\n",
        "  # print(extracted_list, output)\n",
        "  for out in extracted_list:\n",
        "    # print(out)\n",
        "    namelist1[out[0].lower()] = out[1]\n",
        "\n",
        "  namelist2 = {}\n",
        "  for out in output:\n",
        "    # if out['score'] > 0.70:\n",
        "      # print(\"printing out\", out)\n",
        "      word = findword(text_list[0].lower(), out['start'], out['end'])\n",
        "      # print(word)\n",
        "      namelist2[word] =  out['score']\n",
        "\n",
        "  # print(namelist2)\n",
        "  # print(namelist1)\n",
        "  result = {}\n",
        "  for place in namelist2.keys():\n",
        "    if is_hindi(place):\n",
        "      continue\n",
        "    if place not in namelist1.keys():\n",
        "      result[place] = \"Looks Like\"\n",
        "\n",
        "    elif namelist1[place] == 'PLACE':\n",
        "      result[place] = \"Certain\"\n",
        "    else:\n",
        "      result[place] = f'{place} occurs in the context of {namelist1[place]} but its name may resemble the name of a place'\n",
        "\n",
        "  for place in namelist1.keys():\n",
        "    if namelist1[place] == 'PLACE' and place not in result.keys():\n",
        "      result[place] = \"Most Likely\"\n",
        "      if len(place.split(' ')) > 1:\n",
        "        for x in place.split(' '):\n",
        "          if x in result.keys():\n",
        "            del result[x]\n",
        "\n",
        "\n",
        "  return result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "9lsRMBgB0OBm"
      },
      "outputs": [],
      "source": [
        "def getNER(text):\n",
        "  text = text.capitalize()\n",
        "  # print(text)\n",
        "  text_list = [text]\n",
        "  df = spark.createDataFrame(text_list, StringType()).toDF(\"text\")\n",
        "  result = nlp_pipeline.fit(df).transform(df)\n",
        "\n",
        "\n",
        "  extracted_result = result.select(F.explode(F.arrays_zip(result.ner_chunk.result, result.ner_chunk.metadata)).alias(\"cols\")) \\\n",
        "    .select(F.expr(\"cols['0']\").alias(\"chunk\"),\n",
        "            F.expr(\"cols['1']['entity']\").alias(\"ner_label\"))\n",
        "  # extracted_result.show(truncate=False)\n",
        "\n",
        "  extracted_list = extracted_result.collect()\n",
        "\n",
        "  checkpoint = \"/content/drive/MyDrive/NER/checkpoint-3135\"\n",
        "  token_classifier = pipeline(\n",
        "      \"token-classification\", model=checkpoint, aggregation_strategy=\"simple\"\n",
        "  )\n",
        "\n",
        "  output = token_classifier(text_list[0].lower())\n",
        "  # print(extracted_list, output)\n",
        "  answer = combinedOutput(extracted_list, output, text_list)\n",
        "  return answer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aCtJ2op21PZs",
        "outputId": "0cfc0566-a6b9-4a95-e0d9-1e70d36ca923"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'surat': 'Certain', 'dujrat': 'Certain'}"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "getNER(\"surat is a city in dujrat\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "RZr3Anr1q6k1"
      },
      "outputs": [],
      "source": [
        "# The function to find similarity between correctAnswer and userAnswer, based on levenshtein distance\n",
        "def validate_answer_levenshtein(correct_answer, user_answer, threshold=80):\n",
        "    distance = Levenshtein.distance(user_answer.lower(), correct_answer.lower())\n",
        "    similarity = 1 - (distance / max(len(user_answer), len(correct_answer)))\n",
        "    # normalising from [0,1] to [0,100]\n",
        "    return round(100*similarity, 2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "XiQrxJI_qNxy"
      },
      "outputs": [],
      "source": [
        "def fuzzyMatching(fuzzy_li, universe_of_names):\n",
        "    global_nearest_match_li = []\n",
        "    max_similarity = 0\n",
        "\n",
        "    for word in fuzzy_li:\n",
        "        nearest_match_li = []\n",
        "        n = 0\n",
        "        for checking_word, type_ in universe_of_names:\n",
        "            temp = validate_answer_levenshtein(word, checking_word)\n",
        "            if n == 0:\n",
        "                nearest_match_li.append((checking_word, temp, type_))\n",
        "                n += 1\n",
        "            else:\n",
        "                for i in range(n):\n",
        "                    # print(len(nearest_match_li), n)\n",
        "                    if temp >= nearest_match_li[i][1]:\n",
        "                        nearest_match_li.insert(i, (checking_word, temp, type_))\n",
        "                        n += 1\n",
        "                        break\n",
        "                else:\n",
        "                    nearest_match_li.append((checking_word, temp, type_))\n",
        "                    n += 1\n",
        "            if n == 4:\n",
        "                nearest_match_li = nearest_match_li[:-1]\n",
        "                n -= 1\n",
        "        global_nearest_match_li.append(nearest_match_li)\n",
        "\n",
        "    return global_nearest_match_li"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "TgyE9FXu40ew"
      },
      "outputs": [],
      "source": [
        "def fuzzyMatchingPreprocessing():\n",
        "\n",
        "    countries_df = pd.read_csv(\"/content/drive/MyDrive/NER/countries.csv\")\n",
        "    states_df = pd.read_csv(\"/content/drive/MyDrive/NER/states.csv\")\n",
        "    cities_df = pd.read_csv(\"/content/drive/MyDrive/NER/cities.csv\")\n",
        "\n",
        "\n",
        "    n_cities = len(cities_df)\n",
        "    n_states = len(states_df)\n",
        "    n_countries = len(countries_df)\n",
        "\n",
        "    cities_list = [(cities_df.iloc[i][\"name\"].lower(), \"city\") for i in range(n_cities)]\n",
        "    states_list = [(states_df.iloc[i][\"name\"].lower(), \"state\") for i in range(n_states)]\n",
        "    countries_list = [(countries_df.iloc[i][\"name\"].lower(), \"country\") for i in range(n_countries)]\n",
        "\n",
        "    universe_of_names = countries_list + states_list + cities_list\n",
        "    return universe_of_names"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "vwXlmEu8ILK6"
      },
      "outputs": [],
      "source": [
        "def fuzzyMatchingComplete (fuzzy_li, universe_of_names):\n",
        "\n",
        "    final_list = fuzzyMatching(fuzzy_li, universe_of_names)\n",
        "\n",
        "    return final_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "8Bq6zVDVCSit"
      },
      "outputs": [],
      "source": [
        "def add(word,entity,universe_of_names):\n",
        "  entry = (word.lower(), entity.lower())\n",
        "  if entry in universe_of_names:\n",
        "      print(entry, \" is already present in the database\")\n",
        "  else:\n",
        "      universe_of_names.append(entry)\n",
        "  return universe_of_names\n",
        "\n",
        "def delete(word,entity,universe_of_names):\n",
        "  entry = (word.lower(), entity.lower())\n",
        "  if entry not in universe_of_names:\n",
        "    print(tuple([word,entity.lower()]), \" is not present in the database\")\n",
        "  else:\n",
        "    universe_of_names.remove(entry)\n",
        "  return universe_of_names"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "si6d5ELgExUm"
      },
      "outputs": [],
      "source": [
        "universe_of_names = fuzzyMatchingPreprocessing()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "8gHLWIL3I04x"
      },
      "outputs": [],
      "source": [
        "def main(universe_of_names):\n",
        "  while True:\n",
        "      # Take input from the user\n",
        "      print(\"\")\n",
        "\n",
        "      request_type = input(\"Enter a request type: (Entering sentence: 1, add to database: 2, delete to database: 3), (exit: -1) : \")\n",
        "      print(\"\")\n",
        "      if request_type == '-1':\n",
        "            print(\"Exiting...\")\n",
        "            break\n",
        "      elif request_type == '1' :\n",
        "        user_input = input(\"Enter a sentence (type '-1' to exit): \")\n",
        "\n",
        "        # Check if the user wants to exit\n",
        "        if user_input == '-1':\n",
        "            print(\"Exiting...\")\n",
        "            break\n",
        "\n",
        "        # Call the processing function and display the result\n",
        "        result = getNER(user_input)\n",
        "        # print(f\"Result: {result}\")\n",
        "\n",
        "        matched_cities = fuzzyMatchingComplete(result.keys(), universe_of_names)\n",
        "\n",
        "        print(result)\n",
        "        print()\n",
        "        i=0\n",
        "        for entity in result.keys():\n",
        "          print(\"matching places for \", entity, \" are : \", matched_cities[i])\n",
        "          i+=1\n",
        "          print()\n",
        "      elif request_type == '2':\n",
        "        user_input = input(\"Enter the name of the place (type '-1' to exit): \")\n",
        "        print(\"\")\n",
        "        user_category = input(\"Enter the type of place(city, state, country)  (type '-1' to exit): \")\n",
        "        print(\"\")\n",
        "        if user_input == '-1' or user_category == '-1':\n",
        "          break\n",
        "        universe_of_names = add(user_input, user_category, universe_of_names)\n",
        "\n",
        "      elif request_type == \"3\":\n",
        "        user_input = input(\"Enter the name of the place  (type '-1' to exit): \")\n",
        "        print(\"\")\n",
        "        user_category = input(\"Enter the type of place(city, state, country)  (type '-1' to exit): \")\n",
        "        print(\"\")\n",
        "        if user_input == '-1' or user_category == '-1':\n",
        "          break\n",
        "        universe_of_names = delete(user_input, user_category, universe_of_names)\n",
        "      else:\n",
        "        print(\"Please enter a valid input\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DZ8b2vFK09qo",
        "outputId": "40dacbb6-db1d-453c-b428-a306adbc962f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Enter a request type: (Entering sentence: 1, add to database: 2, delete to database: 3), (exit: -1) : 1\n",
            "\n",
            "Enter a sentence (type '-1' to exit): Where is the best place to visit forchristmas, New York or Kashmir ?\n",
            "{'new york': 'Most Likely', 'kashmir': 'Most Likely'}\n",
            "\n",
            "matching places for  new york  are :  [('new york', 100.0, 'state'), ('new works', 77.78, 'city'), ('new dorp', 75.0, 'city')]\n",
            "\n",
            "matching places for  kashmir  are :  [('kashmor', 85.71, 'city'), ('kashmar', 85.71, 'city'), ('kashin', 71.43, 'city')]\n",
            "\n",
            "\n",
            "Enter a request type: (Entering sentence: 1, add to database: 2, delete to database: 3), (exit: -1) : which is hotter, Surat or ajmir?\n",
            "\n",
            "Please enter a valid input\n",
            "\n",
            "Enter a request type: (Entering sentence: 1, add to database: 2, delete to database: 3), (exit: -1) : 1\n",
            "\n",
            "Enter a sentence (type '-1' to exit): which is hotter, Surat or ajmir?\n",
            "{'surat': 'Looks Like', 'ajmir': 'Looks Like'}\n",
            "\n",
            "matching places for  surat  are :  [('surat', 100.0, 'city'), ('suratá', 83.33, 'city'), ('sura', 80.0, 'city')]\n",
            "\n",
            "matching places for  ajmir  are :  [('ajmer', 80.0, 'city'), ('ajamil', 66.67, 'city'), ('cujmir', 66.67, 'city')]\n",
            "\n",
            "\n",
            "Enter a request type: (Entering sentence: 1, add to database: 2, delete to database: 3), (exit: -1) : -1\n",
            "\n",
            "Exiting...\n"
          ]
        }
      ],
      "source": [
        "main(universe_of_names)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "interpreter": {
      "hash": "45150093197569bb3a58481dcd32cd1adb45462fa3448719e8ac38ada6166aca"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}