nlp_tpot.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O-7HqNRCgMc9"
      },
      "source": [
        "# Name: Jayveersinh Raj\n",
        "# Group: BS20-DS-01\n",
        "# Email: j.raj@innopolis.university"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oqSW-jw5bOA6"
      },
      "source": [
        "#Importing libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gXhSKUiFbRfN"
      },
      "outputs": [],
      "source": [
        "!pip install tpot\n",
        "!pip install scikit-mdr\n",
        "!pip install nltk"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eDRjz0nfHZ7B"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "df = pd.read_csv(\"/content/NIC GP data.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "A5NUDwzKHrmn",
        "outputId": "c0fd347a-8311-488a-9e03-34c1967d5d51"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c679bd47-806d-4522-8a40-c60e0c358426\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>id</th>\n",
              "      <th>qid1</th>\n",
              "      <th>qid2</th>\n",
              "      <th>question1</th>\n",
              "      <th>question2</th>\n",
              "      <th>bot?</th>\n",
              "      <th>number_likes</th>\n",
              "      <th>is_duplicate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>What is the step by step guide to invest in sh...</td>\n",
              "      <td>What is the step by step guide to invest in sh...</td>\n",
              "      <td>False</td>\n",
              "      <td>30634</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
              "      <td>What would happen if the Indian government sto...</td>\n",
              "      <td>False</td>\n",
              "      <td>64192</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>How can I increase the speed of my internet co...</td>\n",
              "      <td>How can Internet speed be increased by hacking...</td>\n",
              "      <td>False</td>\n",
              "      <td>1396</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>7</td>\n",
              "      <td>8</td>\n",
              "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
              "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
              "      <td>False</td>\n",
              "      <td>24049</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>9</td>\n",
              "      <td>10</td>\n",
              "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
              "      <td>Which fish would survive in salt water?</td>\n",
              "      <td>False</td>\n",
              "      <td>13076</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c679bd47-806d-4522-8a40-c60e0c358426')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c679bd47-806d-4522-8a40-c60e0c358426 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c679bd47-806d-4522-8a40-c60e0c358426');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "   Unnamed: 0  id  qid1  qid2  \\\n",
              "0           0   0     1     2   \n",
              "1           1   1     3     4   \n",
              "2           2   2     5     6   \n",
              "3           3   3     7     8   \n",
              "4           4   4     9    10   \n",
              "\n",
              "                                           question1  \\\n",
              "0  What is the step by step guide to invest in sh...   \n",
              "1  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
              "2  How can I increase the speed of my internet co...   \n",
              "3  Why am I mentally very lonely? How can I solve...   \n",
              "4  Which one dissolve in water quikly sugar, salt...   \n",
              "\n",
              "                                           question2   bot?  number_likes  \\\n",
              "0  What is the step by step guide to invest in sh...  False         30634   \n",
              "1  What would happen if the Indian government sto...  False         64192   \n",
              "2  How can Internet speed be increased by hacking...  False          1396   \n",
              "3  Find the remainder when [math]23^{24}[/math] i...  False         24049   \n",
              "4            Which fish would survive in salt water?  False         13076   \n",
              "\n",
              "   is_duplicate  \n",
              "0             0  \n",
              "1             0  \n",
              "2             0  \n",
              "3             0  \n",
              "4             0  "
            ]
          },
          "execution_count": 74,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XUWoEsQ1Ht-1"
      },
      "outputs": [],
      "source": [
        "df = df.drop(df.iloc[:, :4], axis = 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "2ZB6TzIQLggF",
        "outputId": "8e350454-db01-4cac-83a1-0c3bc9182b4a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-409cbafa-9e11-4e04-8ec9-06f9d1380c6c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question1</th>\n",
              "      <th>question2</th>\n",
              "      <th>bot?</th>\n",
              "      <th>number_likes</th>\n",
              "      <th>is_duplicate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What is the step by step guide to invest in sh...</td>\n",
              "      <td>What is the step by step guide to invest in sh...</td>\n",
              "      <td>False</td>\n",
              "      <td>30634</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
              "      <td>What would happen if the Indian government sto...</td>\n",
              "      <td>False</td>\n",
              "      <td>64192</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How can I increase the speed of my internet co...</td>\n",
              "      <td>How can Internet speed be increased by hacking...</td>\n",
              "      <td>False</td>\n",
              "      <td>1396</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
              "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
              "      <td>False</td>\n",
              "      <td>24049</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
              "      <td>Which fish would survive in salt water?</td>\n",
              "      <td>False</td>\n",
              "      <td>13076</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-409cbafa-9e11-4e04-8ec9-06f9d1380c6c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-409cbafa-9e11-4e04-8ec9-06f9d1380c6c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-409cbafa-9e11-4e04-8ec9-06f9d1380c6c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                           question1  \\\n",
              "0  What is the step by step guide to invest in sh...   \n",
              "1  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
              "2  How can I increase the speed of my internet co...   \n",
              "3  Why am I mentally very lonely? How can I solve...   \n",
              "4  Which one dissolve in water quikly sugar, salt...   \n",
              "\n",
              "                                           question2   bot?  number_likes  \\\n",
              "0  What is the step by step guide to invest in sh...  False         30634   \n",
              "1  What would happen if the Indian government sto...  False         64192   \n",
              "2  How can Internet speed be increased by hacking...  False          1396   \n",
              "3  Find the remainder when [math]23^{24}[/math] i...  False         24049   \n",
              "4            Which fish would survive in salt water?  False         13076   \n",
              "\n",
              "   is_duplicate  \n",
              "0             0  \n",
              "1             0  \n",
              "2             0  \n",
              "3             0  \n",
              "4             0  "
            ]
          },
          "execution_count": 76,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "s1D8WrPcoJIp",
        "outputId": "4f673308-12e4-4929-a3b9-15378b0b93f1"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'What are some of the best films and TV shows that feature strong female protagonists?'"
            ]
          },
          "execution_count": 77,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import random\n",
        "random.choice(df['question1'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ub8_a3P4bW80"
      },
      "source": [
        "# Text preprocessing function that would be called by tokenizers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AvuPxAtpm6B2"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "\n",
        "def replace_symbols(s):\n",
        "    s = s.replace('₹', ' rupee ')\n",
        "    s = s.replace('$', ' dollar ')\n",
        "    s = s.replace('€', ' euro ')\n",
        "    s = s.replace('%', ' percent')\n",
        "    s = s.replace('@', ' at ')\n",
        "    return s\n",
        "\n",
        "def sentence_preprocess(q):\n",
        "    \n",
        "    # Words to lower case\n",
        "    q = str(q).lower().strip()\n",
        "    \n",
        "    # Replace certain special characters with their string equivalents\n",
        "    q = replace_symbols(q)\n",
        "    \n",
        "    # The pattern '[math]' appears around 900 times in the whole dataset.\n",
        "    q = q.replace('[math]', '')\n",
        "    \n",
        "    # Replacing numbers with string equivalents (could be made better by considering for more cases)\n",
        "    q = q.replace(',000,000,000 ', 'b ')\n",
        "    q = q.replace(',000,000 ', 'm ')\n",
        "    q = q.replace(',000 ', 'k ')\n",
        "    q = re.sub(r'([0-9]+)000000000', r'\\1b', q)\n",
        "    q = re.sub(r'([0-9]+)000000', r'\\1m', q)\n",
        "    q = re.sub(r'([0-9]+)000', r'\\1k', q)\n",
        "    \n",
        "    # Decontracting words\n",
        "    # https://en.wikipedia.org/wiki/Wikipedia%3aList_of_English_contractions\n",
        "    # https://stackoverflow.com/a/19794953\n",
        "    contractions = { \n",
        "    \"ain't\": \"am not\",\n",
        "    \"aren't\": \"are not\",\n",
        "    \"can't\": \"can not\",\n",
        "    \"can't've\": \"can not have\",\n",
        "    \"'cause\": \"because\",\n",
        "    \"could've\": \"could have\",\n",
        "    \"couldn't\": \"could not\",\n",
        "    \"couldn't've\": \"could not have\",\n",
        "    \"didn't\": \"did not\",\n",
        "    \"doesn't\": \"does not\",\n",
        "    \"don't\": \"do not\",\n",
        "    \"hadn't\": \"had not\",\n",
        "    \"hadn't've\": \"had not have\",\n",
        "    \"hasn't\": \"has not\",\n",
        "    \"haven't\": \"have not\",\n",
        "    \"he'd\": \"he would\",\n",
        "    \"he'd've\": \"he would have\",\n",
        "    \"he'll\": \"he will\",\n",
        "    \"he'll've\": \"he will have\",\n",
        "    \"he's\": \"he is\",\n",
        "    \"how'd\": \"how did\",\n",
        "    \"how'd'y\": \"how do you\",\n",
        "    \"how'll\": \"how will\",\n",
        "    \"how's\": \"how is\",\n",
        "    \"i'd\": \"i would\",\n",
        "    \"i'd've\": \"i would have\",\n",
        "    \"i'll\": \"i will\",\n",
        "    \"i'll've\": \"i will have\",\n",
        "    \"i'm\": \"i am\",\n",
        "    \"i've\": \"i have\",\n",
        "    \"isn't\": \"is not\",\n",
        "    \"it'd\": \"it would\",\n",
        "    \"it'd've\": \"it would have\",\n",
        "    \"it'll\": \"it will\",\n",
        "    \"it'll've\": \"it will have\",\n",
        "    \"it's\": \"it is\",\n",
        "    \"let's\": \"let us\",\n",
        "    \"ma'am\": \"madam\",\n",
        "    \"mayn't\": \"may not\",\n",
        "    \"might've\": \"might have\",\n",
        "    \"mightn't\": \"might not\",\n",
        "    \"mightn't've\": \"might not have\",\n",
        "    \"must've\": \"must have\",\n",
        "    \"mustn't\": \"must not\",\n",
        "    \"mustn't've\": \"must not have\",\n",
        "    \"needn't\": \"need not\",\n",
        "    \"needn't've\": \"need not have\",\n",
        "    \"o'clock\": \"of the clock\",\n",
        "    \"oughtn't\": \"ought not\",\n",
        "    \"oughtn't've\": \"ought not have\",\n",
        "    \"shan't\": \"shall not\",\n",
        "    \"sha'n't\": \"shall not\",\n",
        "    \"shan't've\": \"shall not have\",\n",
        "    \"she'd\": \"she would\",\n",
        "    \"she'd've\": \"she would have\",\n",
        "    \"she'll\": \"she will\",\n",
        "    \"she'll've\": \"she will have\",\n",
        "    \"she's\": \"she is\",\n",
        "    \"should've\": \"should have\",\n",
        "    \"shouldn't\": \"should not\",\n",
        "    \"shouldn't've\": \"should not have\",\n",
        "    \"so've\": \"so have\",\n",
        "    \"so's\": \"so as\",\n",
        "    \"that'd\": \"that would\",\n",
        "    \"that'd've\": \"that would have\",\n",
        "    \"that's\": \"that is\",\n",
        "    \"there'd\": \"there would\",\n",
        "    \"there'd've\": \"there would have\",\n",
        "    \"there's\": \"there is\",\n",
        "    \"they'd\": \"they would\",\n",
        "    \"they'd've\": \"they would have\",\n",
        "    \"they'll\": \"they will\",\n",
        "    \"they'll've\": \"they will have\",\n",
        "    \"they're\": \"they are\",\n",
        "    \"they've\": \"they have\",\n",
        "    \"to've\": \"to have\",\n",
        "    \"wasn't\": \"was not\",\n",
        "    \"we'd\": \"we would\",\n",
        "    \"we'd've\": \"we would have\",\n",
        "    \"we'll\": \"we will\",\n",
        "    \"we'll've\": \"we will have\",\n",
        "    \"we're\": \"we are\",\n",
        "    \"we've\": \"we have\",\n",
        "    \"weren't\": \"were not\",\n",
        "    \"what'll\": \"what will\",\n",
        "    \"what'll've\": \"what will have\",\n",
        "    \"what're\": \"what are\",\n",
        "    \"what's\": \"what is\",\n",
        "    \"what've\": \"what have\",\n",
        "    \"when's\": \"when is\",\n",
        "    \"when've\": \"when have\",\n",
        "    \"where'd\": \"where did\",\n",
        "    \"where's\": \"where is\",\n",
        "    \"where've\": \"where have\",\n",
        "    \"who'll\": \"who will\",\n",
        "    \"who'll've\": \"who will have\",\n",
        "    \"who's\": \"who is\",\n",
        "    \"who've\": \"who have\",\n",
        "    \"why's\": \"why is\",\n",
        "    \"why've\": \"why have\",\n",
        "    \"will've\": \"will have\",\n",
        "    \"won't\": \"will not\",\n",
        "    \"won't've\": \"will not have\",\n",
        "    \"would've\": \"would have\",\n",
        "    \"wouldn't\": \"would not\",\n",
        "    \"wouldn't've\": \"would not have\",\n",
        "    \"y'all\": \"you all\",\n",
        "    \"y'all'd\": \"you all would\",\n",
        "    \"y'all'd've\": \"you all would have\",\n",
        "    \"y'all're\": \"you all are\",\n",
        "    \"y'all've\": \"you all have\",\n",
        "    \"you'd\": \"you would\",\n",
        "    \"you'd've\": \"you would have\",\n",
        "    \"you'll\": \"you will\",\n",
        "    \"you'll've\": \"you will have\",\n",
        "    \"you're\": \"you are\",\n",
        "    \"you've\": \"you have\",\n",
        "\n",
        "    # added extra by me\n",
        "    \"ve \": \" have\",\n",
        "    \"n't\": \" not\",\n",
        "    \"re\" : \" are\",\n",
        "    \"ll\" : \" will\"\n",
        "    }\n",
        "\n",
        "    decontracted = []\n",
        "\n",
        "    for word in q.split():\n",
        "        if word in contractions:\n",
        "            word = contractions[word]\n",
        "\n",
        "        decontracted.append(word)\n",
        "\n",
        "    q = ' '.join(decontracted)\n",
        "    \n",
        "    # Remove HTML tags\n",
        "    clean = re.compile('<.*?>')\n",
        "    q = re.sub(clean, '', q)\n",
        "\n",
        "    # Remove punctuations\n",
        "    pattern = re.compile('\\W')\n",
        "    q = re.sub(pattern, ' ', q).strip()\n",
        "\n",
        "    \n",
        "    return q"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7MDUsvYPbiIu"
      },
      "source": [
        "# Separating labels in y, and numberical features in X"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jwiG6uK2QsN8"
      },
      "outputs": [],
      "source": [
        "X = df.iloc[:, 2:-1]\n",
        "y = df.iloc[:, -1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "qCA8zqD6CMcD",
        "outputId": "c95d1bb2-59f6-4a30-fa14-e7cc142dad22"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-47656605-49fd-492f-b361-ff4c4c612508\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question1</th>\n",
              "      <th>question2</th>\n",
              "      <th>bot?</th>\n",
              "      <th>number_likes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>what is the step by step guide to invest in sh...</td>\n",
              "      <td>what is the step by step guide to invest in sh...</td>\n",
              "      <td>False</td>\n",
              "      <td>30634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>what is the story of kohinoor koh i noor diamond</td>\n",
              "      <td>what would happen if the indian government sto...</td>\n",
              "      <td>False</td>\n",
              "      <td>64192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>how can i increase the speed of my internet co...</td>\n",
              "      <td>how can internet speed be increased by hacking...</td>\n",
              "      <td>False</td>\n",
              "      <td>1396</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>why am i mentally very lonely how can i solve it</td>\n",
              "      <td>find the remainder when 23 24 math is divided ...</td>\n",
              "      <td>False</td>\n",
              "      <td>24049</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>which one dissolve in water quikly sugar salt ...</td>\n",
              "      <td>which fish would survive in salt water</td>\n",
              "      <td>False</td>\n",
              "      <td>13076</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1995</th>\n",
              "      <td>i am visiting sri lanka soonfor 9 days how can...</td>\n",
              "      <td>do indians hate sri lankans</td>\n",
              "      <td>False</td>\n",
              "      <td>24454</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1996</th>\n",
              "      <td>what are some good examples of 4 stanza poems</td>\n",
              "      <td>what are some good ilocano poems</td>\n",
              "      <td>False</td>\n",
              "      <td>2611</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1997</th>\n",
              "      <td>which cpu is better i3 4th gen or 6th gen</td>\n",
              "      <td>which is better intel i5 6th gen or i7 5th gen</td>\n",
              "      <td>True</td>\n",
              "      <td>18483</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1998</th>\n",
              "      <td>what are some of the best tourist places to vi...</td>\n",
              "      <td>where are the foremost tourist places in chhat...</td>\n",
              "      <td>True</td>\n",
              "      <td>28018</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1999</th>\n",
              "      <td>what are the differences between a love marria...</td>\n",
              "      <td>which is better an arranged marriage or a love...</td>\n",
              "      <td>False</td>\n",
              "      <td>20559</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2000 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-47656605-49fd-492f-b361-ff4c4c612508')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-47656605-49fd-492f-b361-ff4c4c612508 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-47656605-49fd-492f-b361-ff4c4c612508');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                                              question1  \\\n",
              "0     what is the step by step guide to invest in sh...   \n",
              "1      what is the story of kohinoor koh i noor diamond   \n",
              "2     how can i increase the speed of my internet co...   \n",
              "3      why am i mentally very lonely how can i solve it   \n",
              "4     which one dissolve in water quikly sugar salt ...   \n",
              "...                                                 ...   \n",
              "1995  i am visiting sri lanka soonfor 9 days how can...   \n",
              "1996      what are some good examples of 4 stanza poems   \n",
              "1997          which cpu is better i3 4th gen or 6th gen   \n",
              "1998  what are some of the best tourist places to vi...   \n",
              "1999  what are the differences between a love marria...   \n",
              "\n",
              "                                              question2   bot?  number_likes  \n",
              "0     what is the step by step guide to invest in sh...  False         30634  \n",
              "1     what would happen if the indian government sto...  False         64192  \n",
              "2     how can internet speed be increased by hacking...  False          1396  \n",
              "3     find the remainder when 23 24 math is divided ...  False         24049  \n",
              "4                which fish would survive in salt water  False         13076  \n",
              "...                                                 ...    ...           ...  \n",
              "1995                        do indians hate sri lankans  False         24454  \n",
              "1996                   what are some good ilocano poems  False          2611  \n",
              "1997     which is better intel i5 6th gen or i7 5th gen   True         18483  \n",
              "1998  where are the foremost tourist places in chhat...   True         28018  \n",
              "1999  which is better an arranged marriage or a love...  False         20559  \n",
              "\n",
              "[2000 rows x 4 columns]"
            ]
          },
          "execution_count": 191,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.iloc[:, :-1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EHtdFziQCGML",
        "outputId": "493f7ce5-7fcb-497c-9ca9-c8bd8e48a065"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0       0\n",
              "1       0\n",
              "2       0\n",
              "3       0\n",
              "4       0\n",
              "       ..\n",
              "1995    0\n",
              "1996    0\n",
              "1997    0\n",
              "1998    1\n",
              "1999    0\n",
              "Name: is_duplicate, Length: 2000, dtype: int64"
            ]
          },
          "execution_count": 187,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6gvyg-DV_FKs",
        "outputId": "3a2c251f-a290-41fc-bf66-93e66859d43f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "pandas.core.series.Series"
            ]
          },
          "execution_count": 159,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "type(y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "67KukjH0Ueb7",
        "outputId": "1e6ebb74-801a-4c22-d8b9-abb77bb62b4e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0       0\n",
              "1       0\n",
              "2       0\n",
              "3       0\n",
              "4       0\n",
              "       ..\n",
              "1995    0\n",
              "1996    0\n",
              "1997    0\n",
              "1998    1\n",
              "1999    0\n",
              "Name: is_duplicate, Length: 2000, dtype: int64"
            ]
          },
          "execution_count": 92,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uH0n4bh88mVG",
        "outputId": "fae73e91-9bd9-47b3-82c3-93625dcf2757"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    1258\n",
              "1     742\n",
              "Name: is_duplicate, dtype: int64"
            ]
          },
          "execution_count": 104,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "y.value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qVwG_pp5cIkc"
      },
      "source": [
        "# Function that will be used by word2vec using nltk for word2vec encoding, it will tokenize the words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_96oaVaPzvJw",
        "outputId": "3052c00c-5b5b-41ef-9659-f525f8950fcc"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords              \n",
        "from nltk import word_tokenize, sent_tokenize\n",
        "import string\n",
        "\n",
        "nltk.download('punkt')\n",
        "\n",
        "def words_list(questions):\n",
        "  q_as_words = []\n",
        "  \n",
        "  for question in questions:\n",
        "    tokens = word_tokenize(question)   \n",
        "    \n",
        "    #convert to lower case\n",
        "    tokens = [w.lower() for w in tokens]\n",
        "\n",
        "    table = str.maketrans('','', string.punctuation)\n",
        "\n",
        "    stripped = [w.translate(table) for w in tokens]\n",
        "     \n",
        "    # remove remaining tokens that are not alphabetic\n",
        "    words = [w for w in stripped if w.isalpha()]\n",
        "\n",
        "    q_as_words.append(words)\n",
        " \n",
        "  return q_as_words"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XISz-a6Xcc_A"
      },
      "source": [
        "# Function that word2vec will use for encoding and embedding using gensim"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y72Ah7fP3u5-",
        "outputId": "16c21bbd-eb2a-4c0c-e175-d4b1127e3b71"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.7/dist-packages (4.2.0)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (5.2.1)\n",
            "Requirement already satisfied: numpy>=1.17.0 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.21.6)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.7/dist-packages (from gensim) (1.7.3)\n"
          ]
        }
      ],
      "source": [
        "!pip install gensim --upgrade"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eh4dWp0Xx7kD"
      },
      "outputs": [],
      "source": [
        "import gensim\n",
        "import math\n",
        "from tensorflow.keras.preprocessing.text import one_hot, Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "def word2vec(questions):\n",
        "\n",
        "  processed_sent = words_list(questions)   \n",
        "  # now training embeddings for each word \n",
        "  model_1 = gensim.models.Word2Vec(sentences = processed_sent, vector_size = len(questions), min_count=1, window=5, workers=4 )\n",
        "  \n",
        "  # to get total number of unique words\n",
        "  words = list(model_1.wv.key_to_index)\n",
        "  \n",
        "  leng = 0\n",
        "  length = [(leng + len(x)) for x in processed_sent]\n",
        " \n",
        "  avg_length = sum(length)/len(questions)\n",
        "\n",
        "  max_len = math.ceil(avg_length)\n",
        "  \n",
        "  #keeps 6000 most common words\n",
        "  max_features = 6000  \n",
        "  tokenizer = Tokenizer(num_words=max_features)      \n",
        "                  \n",
        "  tokenizer.fit_on_texts(processed_sent)\n",
        "  sequence = tokenizer.texts_to_sequences(processed_sent)\n",
        "  processed_sent = pad_sequences(sequence, maxlen = max_len)\n",
        "\n",
        "  # Preparing embedding matrix\n",
        "  vocab_size = len(tokenizer.word_index)+1\n",
        "  embedding_matrix = np.zeros((vocab_size, len(questions)))\n",
        "  \n",
        "  for  word, i in tokenizer.word_index.items():\n",
        "      embedding_vector = model_1.wv[word]\n",
        "      if embedding_vector is not None:\n",
        "          embedding_matrix[i] = embedding_vector\n",
        "\n",
        "  return embedding_matrix"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W2w6BNWUckaM"
      },
      "source": [
        "# Testing word2vec function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "n34SjfSJ5JnD"
      },
      "outputs": [],
      "source": [
        "q1_list = df['question1'].apply(sentence_preprocess).tolist()\n",
        "q2_list = df['question2'].apply(sentence_preprocess).tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QX7_43pv18jY"
      },
      "outputs": [],
      "source": [
        "bag1 = word2vec(q1_list)\n",
        "bag2 = word2vec(q2_list)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bi8_Zkyp7sRB"
      },
      "outputs": [],
      "source": [
        "bag1 = bag1.T\n",
        "bag2 = bag2.T"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TbbOLhknE1Xg"
      },
      "outputs": [],
      "source": [
        "con = np.concatenate((bag1,bag2), axis = 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bcvvEQXfE9k5",
        "outputId": "24afa03c-1243-4198-c151-e41cd7f8fe9e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2000, 9037)"
            ]
          },
          "execution_count": 113,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "con.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X3SYSSJOCKCO",
        "outputId": "d4c61c13-b7af-4088-c482-28d23859522b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "((2000, 4592), (2000, 4445))"
            ]
          },
          "execution_count": 114,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bag1.shape, bag2.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "htQMhSSu8RFy",
        "outputId": "27d6061f-66a9-4c14-da59-4283f9de03c2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "numpy.ndarray"
            ]
          },
          "execution_count": 115,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "type(bag1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mMz-F43gc6mz"
      },
      "source": [
        "# CustomPreprocess class that would be used by tpot for text preprocessing\n",
        "3 Text preprocessing methods are used:\n",
        "* TfidfVectorizer\n",
        "* CountVectorizer\n",
        "* word2vec"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oRse3sp-dkuo"
      },
      "source": [
        "# MDR was used after getting the texts to vectors. Reference [Link](https://github.com/EpistasisLab/scikit-mdr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ro5m1ZuXevC6"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn.base import TransformerMixin, BaseEstimator\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "\n",
        "import copy\n",
        "\n",
        "\n",
        "class CustomPreprocess(TransformerMixin, BaseEstimator):\n",
        "    def __init__(self, name_encoding):\n",
        "        self.name_encoding = name_encoding\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "      return self\n",
        "    \n",
        "    def transform(self, X, y=None):\n",
        "\n",
        "        # Copying dataframe to not change original\n",
        "        X_ = copy.deepcopy(X)\n",
        "        X_.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
        "        X_.fillna(0)\n",
        "\n",
        "        # tfidf\n",
        "        if(self.name_encoding == \"tfidf\"):\n",
        "          # calling preprocess function on texts on do some preprocessing\n",
        "          X_['question1'] =  X_['question1'].apply(sentence_preprocess)\n",
        "          X_['question2'] =  X_['question2'].apply(sentence_preprocess)\n",
        "          \n",
        "          # numerical columns\n",
        "          X_num = copy.deepcopy(X_.iloc[:, 2:-1])\n",
        "          y_ = copy.deepcopy(X_.iloc[:, -1])\n",
        "          \n",
        "          X_num = pd.get_dummies(X_num, columns = [\"bot?\"])\n",
        "          X_num = X_num.drop([\"bot?_False\"], axis = 1)\n",
        "          \n",
        "          # tfidf vectorizer\n",
        "          vectorizer = TfidfVectorizer() \n",
        "          word_vec = vectorizer.fit_transform(X_['question1'])\n",
        "          word_vec2 = vectorizer.fit_transform(X_['question2'])\n",
        "\n",
        "          df_vector = pd.DataFrame(word_vec.toarray()) \n",
        "          df_vector2 = pd.DataFrame(word_vec2.toarray()) \n",
        "          \n",
        "          # concatinate with numerical features\n",
        "          numeric_df = pd.concat([df_vector, df_vector2], axis = 1)\n",
        "          s = pd.concat([numeric_df, X_num], axis = 1)\n",
        "       \n",
        "          # using mdr on recieved vector\n",
        "          from mdr import MDR\n",
        "          my_mdr = MDR()\n",
        "          \n",
        "          features = s.values\n",
        "          label = df.iloc[:, -1].values\n",
        "          my_mdr.fit(features, label)\n",
        "          new = my_mdr.transform(features)\n",
        "          new_df = pd.DataFrame(new)\n",
        "\n",
        "          return new_df\n",
        "          \n",
        "    \n",
        "        # index encoding\n",
        "        elif(self.name_encoding == \"count_vector\"):\n",
        "          # calling preprocess function on texts on do some preprocessing\n",
        "          X_['question1'] =  X_['question1'].apply(sentence_preprocess)\n",
        "          X_['question2'] =  X_['question2'].apply(sentence_preprocess)\n",
        "          \n",
        "          # numerical columns\n",
        "          X_num = copy.deepcopy(X_.iloc[:, 2:-1])\n",
        "          y_ = copy.deepcopy(X_.iloc[:, -1])\n",
        "          \n",
        "          X_num = pd.get_dummies(X_num, columns = [\"bot?\"])\n",
        "          X_num = X_num.drop([\"bot?_False\"], axis = 1)\n",
        "      \n",
        "          # CountVectorizer\n",
        "          vectorizer = CountVectorizer()\n",
        "          word_vec = vectorizer.fit_transform(X_['question1'])\n",
        "          word_vec2 = vectorizer.fit_transform(X_['question2'])\n",
        "\n",
        "          df_vector = pd.DataFrame(word_vec.toarray()) \n",
        "          df_vector2 = pd.DataFrame(word_vec2.toarray()) \n",
        "          \n",
        "          # concatinate with numerical features\n",
        "          numeric_df = pd.concat([df_vector, df_vector2], axis = 1)\n",
        "          s = pd.concat([numeric_df, X_num], axis = 1)\n",
        "      \n",
        "          from mdr import MDR\n",
        "          my_mdr = MDR()\n",
        "          \n",
        "          features = s.values\n",
        "          label = df.iloc[:, -1].values\n",
        "          my_mdr.fit(features, label)\n",
        "          new = my_mdr.transform(features)\n",
        "          \n",
        "          new_df = pd.DataFrame(new)\n",
        "\n",
        "          return new_df\n",
        "          \n",
        "\n",
        "        # word2vec\n",
        "        elif(self.name_encoding == \"word2vec\"):\n",
        "         \n",
        "          # taking numerical columns\n",
        "          X_num = copy.deepcopy(X_.iloc[:, 2:-1])\n",
        "          y_ = copy.deepcopy(X_.iloc[:, -1])\n",
        "          \n",
        "          X_num = pd.get_dummies(X_num, columns = [\"bot?\"])\n",
        "          X_num = X_num.drop([\"bot?_False\"], axis = 1)\n",
        "          \n",
        "          # calling preprocess function on texts on do some preprocessing\n",
        "          q1_list = X_['question1'].apply(sentence_preprocess).tolist()\n",
        "          q2_list = X_['question2'].apply(sentence_preprocess).tolist()\n",
        "          \n",
        "          # calling word2vec preprocessing function\n",
        "          bag1 = word2vec(q1_list)\n",
        "          bag2 = word2vec(q2_list)\n",
        "\n",
        "          bag1 = bag1.T\n",
        "          bag2 = bag2.T\n",
        "          \n",
        "          # concatinating with numerical columns\n",
        "          con = np.concatenate((bag1, bag2, X_num), axis = 1)\n",
        "          s = pd.DataFrame(con)\n",
        "      \n",
        "          # using mdr\n",
        "          from mdr import MDR\n",
        "          my_mdr = MDR()\n",
        "          \n",
        "          features = s.values\n",
        "          label = df.iloc[:, -1].values\n",
        "          my_mdr.fit(features, label)\n",
        "          new = my_mdr.transform(features)\n",
        "\n",
        "          new_df = pd.DataFrame(new)\n",
        "\n",
        "          return new_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mw5OlFsvf-9h"
      },
      "source": [
        "# Driver tpot code cell below"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225,
          "referenced_widgets": [
            "28e6a48f88244937aaf758e18c57350b",
            "e7238681a5ea481d81d9505a678cc7fc",
            "ed8f76a52e8e492bb9945752e0d165f0",
            "9fbd5ea1636d40cd811d29c6a6585577",
            "fce52931aa0444e49bc23d93eaaac4bf",
            "bc3e0d5866664758837269b4ac63075f",
            "1070a627cabd44718ef18466265dc2c6",
            "a9cab66dfd3f4700a84bdb7148cba6bb",
            "e8aee6a6e12443f2a77eea283abcedba",
            "de93ca7d8e8a4d9689cc5eccb15d16da",
            "55ba632d4ce34d5c8704f7f43478d180"
          ]
        },
        "id": "sc2CstNXsCcg",
        "outputId": "16a40eac-14cd-49ba-da6f-ff7669f475be"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "28e6a48f88244937aaf758e18c57350b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Optimization Progress:   0%|          | 0/60 [00:00<?, ?pipeline/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Generation 1 - Current best internal CV score: 0.7507295626052563\n",
            "\n",
            "Generation 2 - Current best internal CV score: 0.7507295626052563\n",
            "\n",
            "Generation 3 - Current best internal CV score: 0.7507295626052563\n",
            "\n",
            "Generation 4 - Current best internal CV score: 0.7507295626052563\n",
            "\n",
            "Generation 5 - Current best internal CV score: 0.7507295626052563\n",
            "\n",
            "Best pipeline: GaussianNB(RBFSampler(VarianceThreshold(CustomPreprocess(input_matrix, name_encoding=word2vec), threshold=0.005), gamma=1.0))\n"
          ]
        }
      ],
      "source": [
        "from tpot import TPOTClassifier\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tpot.config import classifier_config_nn\n",
        "from tpot.config import classifier_config_dict_light\n",
        "\n",
        "from tpot.config import tpot_mdr_classifier_config_dict\n",
        "import copy\n",
        "\n",
        "# X is features without lables\n",
        "X = df.iloc[:, :-1]\n",
        "\n",
        "# using TPOT config\n",
        "config = copy.deepcopy(classifier_config_dict_light)\n",
        "config[\"__main__.CustomPreprocess\"] = {\n",
        "    \"name_encoding\": ['tfidf', 'count_vector' , 'word2vec']\n",
        "}\n",
        "\n",
        "# Model\n",
        "model = TPOTClassifier(generations=5, population_size=10,cv=2,\n",
        "                       early_stop=20,\n",
        "                       scoring='balanced_accuracy',\n",
        "                       config_dict=config, verbosity=2, template = 'CustomPreprocess-Selector-Transformer-Classifier',\n",
        "                       random_state=1, n_jobs=-1)\n",
        "# perform the search\n",
        "model.fit(X, y)\n",
        "\n",
        "# export the best pipeline in a python file\n",
        "model.export('best_pipeline.py')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KSACerQTtR9t"
      },
      "source": [
        "# The one above is the best pipline I got, it would be saved as `best_pipeline.py`. Right now for me it shows `GaussianNB`, and `word2vec` as encoding)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1070a627cabd44718ef18466265dc2c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28e6a48f88244937aaf758e18c57350b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e7238681a5ea481d81d9505a678cc7fc",
              "IPY_MODEL_ed8f76a52e8e492bb9945752e0d165f0",
              "IPY_MODEL_9fbd5ea1636d40cd811d29c6a6585577"
            ],
            "layout": "IPY_MODEL_fce52931aa0444e49bc23d93eaaac4bf"
          }
        },
        "55ba632d4ce34d5c8704f7f43478d180": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9fbd5ea1636d40cd811d29c6a6585577": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_de93ca7d8e8a4d9689cc5eccb15d16da",
            "placeholder": "​",
            "style": "IPY_MODEL_55ba632d4ce34d5c8704f7f43478d180",
            "value": " 60/60 [09:34&lt;00:00,  9.82s/pipeline]"
          }
        },
        "a9cab66dfd3f4700a84bdb7148cba6bb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc3e0d5866664758837269b4ac63075f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "de93ca7d8e8a4d9689cc5eccb15d16da": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7238681a5ea481d81d9505a678cc7fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc3e0d5866664758837269b4ac63075f",
            "placeholder": "​",
            "style": "IPY_MODEL_1070a627cabd44718ef18466265dc2c6",
            "value": "Optimization Progress: 100%"
          }
        },
        "e8aee6a6e12443f2a77eea283abcedba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ed8f76a52e8e492bb9945752e0d165f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a9cab66dfd3f4700a84bdb7148cba6bb",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e8aee6a6e12443f2a77eea283abcedba",
            "value": 60
          }
        },
        "fce52931aa0444e49bc23d93eaaac4bf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}