Customer_Twitter_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "authorship_tag": "ABX9TyNcXnmcmMi/nQyvPZSt/MDL",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/guneet999/Bostonhousepricing/blob/main/Customer_Twitter_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3t9vqc2QCyGm"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "import numpy as pd\n",
        "import re"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q kaggle"
      ],
      "metadata": {
        "id": "IPHFAZsNDse3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir ~/.kaggle"
      ],
      "metadata": {
        "id": "AAtReqU9DGvM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!cp kaggle.json ~/.kaggle/kaggle.json"
      ],
      "metadata": {
        "id": "ndP7Qng8FqU7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!kaggle datasets download -d thoughtvector/customer-support-on-twitter"
      ],
      "metadata": {
        "id": "EuoFegw8Fwku"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip customer-support-on-twitter.zip"
      ],
      "metadata": {
        "id": "iM26uoaNGJlV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Unzipping the data for writing it.\n",
        "from zipfile import ZipFile\n",
        "\n",
        "with ZipFile('customer-support-on-twitter.zip', 'r') as zip_ref:\n",
        "  zip_ref.extractall()\n"
      ],
      "metadata": {
        "id": "Z2Oeg6vgHi8f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Reading the csv file in dataframe\n",
        "\n",
        "df = pd.read_csv('/content/twcs/twcs.csv').astype(str)\n",
        "df.head()\n"
      ],
      "metadata": {
        "id": "xSf0Ha06HjFO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "id": "REa7Hjw0f0XT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**# Let's proceed with text pre-processing the data**"
      ],
      "metadata": {
        "id": "-Mzv7rTFCVw8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**1. Lowecasing**"
      ],
      "metadata": {
        "id": "U7UG4MbIf-aF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df['text'] = df['text'].str.lower()\n",
        "print(df['text'])"
      ],
      "metadata": {
        "id": "5C29bXbFPTmL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check the heads of datasets\n",
        "df.head(2)\n"
      ],
      "metadata": {
        "id": "KlXGca_CbRu2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Converting the complete documents\n",
        "for col in df.select_dtypes(include = ['object']).columns:\n",
        "  df[col] = df[col].str.lower()\n"
      ],
      "metadata": {
        "id": "m1sMgFRub-vq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df[col])"
      ],
      "metadata": {
        "id": "D4sr38RXb_Wj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''for looking the complete list of documents providing iloc, which provides the complete location of all the indexes'''\n",
        "\n",
        "print(df.head(5))"
      ],
      "metadata": {
        "id": "2M3IG9Mkb_gW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**2. Removing HTML Tages**"
      ],
      "metadata": {
        "id": "NFuLoWWcgQZI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def remove_html_tages(text):\n",
        "\n",
        "   clean_text = re.sub(r'<.*?>', '', text)\n",
        "   return clean_text\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "VEQanSk_gS68"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['text'] = df['text'].apply(remove_html_tages)\n",
        "print(df)\n"
      ],
      "metadata": {
        "id": "6kFTyTevgS-U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''# lets remove the tags from the complete dataframe\n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "text_col = ['tweet_id', 'author_id', 'inbound', 'created_at', 'text', 'response_tweet_id', 'in_response_to_tweet_id']\n",
        "print(text_col)\n",
        "\n",
        "\n",
        "def remove_html_tages(text):\n",
        "  soup = BeautifulSoup(text, 'html.parser')\n",
        "  return soup.get_text()\n",
        "\n",
        "for col in text_col:\n",
        "  df[col] = df[col].apply(remove_html_tages)\n",
        "\n",
        "\n",
        "print(df[col])'''"
      ],
      "metadata": {
        "id": "d-WQUZRtgTBb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.head())"
      ],
      "metadata": {
        "id": "AKcq-9ydsYPR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3. Remove HTML URLS**"
      ],
      "metadata": {
        "id": "--hGTgBSbYTh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text_columns = ['tweet_id', 'author_id', 'inbound', 'created_at', 'text', 'response_tweet_id', 'in_response_to_tweet_id']  # Replace with your actual column names\n",
        "\n",
        "# Function to remove URLs\n",
        "def remove_url(text):\n",
        "    pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "    return pattern.sub(\"\", text)\n",
        "\n",
        "# Apply the function to each text column\n",
        "for col in text_columns:\n",
        "    df[col] = df[col].apply(remove_url)\n",
        "\n",
        "# Display a sample of the DataFrame to verify the changes\n",
        "print(df.head())"
      ],
      "metadata": {
        "id": "FIHlXYrEgTEE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.tail())"
      ],
      "metadata": {
        "id": "Lpp-kysagTHQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**from the above we are able to remove the htmls tags and identification can be done from dataframe of head and tail**"
      ],
      "metadata": {
        "id": "6WtIM7uetA48"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4. Punctuation**"
      ],
      "metadata": {
        "id": "pY3NC-H0beI0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import string\n",
        "string.punctuation"
      ],
      "metadata": {
        "id": "VHaP9-_lgTKd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.head())"
      ],
      "metadata": {
        "id": "-C504jy2gTP0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.tail())"
      ],
      "metadata": {
        "id": "H16g0RoiQ3K0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_punctuation(text):\n",
        "    return ''.join(char for char in text if char not in string.punctuation)\n",
        "\n",
        "# Apply the function to each text column\n",
        "for col in text_columns:\n",
        "    df[col] = df[col].apply(remove_punctuation)\n",
        "\n",
        "# Display a sample of the DataFrame to verify the changes\n",
        "print(df.head())"
      ],
      "metadata": {
        "id": "U9ugIpIjNgmX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking whether the punctuations been removed the complete dataset or not.\n",
        "\n",
        "for col in df.columns:\n",
        "    if df[col].dtype == 'object':  # Check if the column contains text data\n",
        "        contains_punctuation = df[col].str.contains(f'[{string.punctuation}]')\n",
        "\n",
        "        # Display the rows where punctuation is present in the current column\n",
        "        rows_with_punctuation = df[contains_punctuation]\n",
        "\n",
        "        # Print information about punctuation presence in the current column\n",
        "        if rows_with_punctuation.empty:\n",
        "            print(f\"No punctuation found in '{col}'.\")\n",
        "        else:\n",
        "            print(f\"Punctuation found in '{col}':\")\n",
        "            print(rows_with_punctuation[[col]])"
      ],
      "metadata": {
        "id": "74K0pVDCq6O_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install emoji==1.7.0"
      ],
      "metadata": {
        "id": "eXbSX9bEvZDv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**5. Remove emojies**"
      ],
      "metadata": {
        "id": "xvXeZ7C9bp6P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import emoji\n",
        "\n",
        "# Assuming 'df' is your DataFrame and 'text_columns' is a list of column names containing text\n",
        "text_columns = ['tweet_id', 'author_id', 'inbound', 'created_at', 'text', 'response_tweet_id', 'in_response_to_tweet_id']  # Replace with your actual column names\n",
        "\n",
        "def remove_emoji(text):\n",
        "    \"\"\"\n",
        "    Remove emojis from a string.\n",
        "    \"\"\"\n",
        "    # Unicode ranges for emojis\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                               u\"\\U0001F700-\\U0001F77F\"  # alchemical symbols\n",
        "                               u\"\\U0001F780-\\U0001F7FF\"  # Geometric Shapes Extended\n",
        "                               u\"\\U0001F800-\\U0001F8FF\"  # Supplemental Arrows-C\n",
        "                               u\"\\U0001F900-\\U0001F9FF\"  # Supplemental Symbols and Pictographs\n",
        "                               u\"\\U0001FA00-\\U0001FA6F\"  # Chess Symbols\n",
        "                               u\"\\U0001FA70-\\U0001FAFF\"  # Symbols and Pictographs Extended-A\n",
        "                               u\"\\U00002702-\\U000027B0\"  # Dingbats\n",
        "                               u\"\\U000024C2-\\U0001F251\"\n",
        "                               \"]+\", flags=re.UNICODE)\n",
        "    clean_text = re.sub(emoji_pattern, '', text)\n",
        "    return clean_text\n",
        "\n",
        "# Apply the remove_emoji function to each text column\n",
        "for col in text_columns:\n",
        "    df[col] = df[col].apply(remove_emoji)\n",
        "\n",
        "\n",
        "print(df.head())\n"
      ],
      "metadata": {
        "id": "MG4m4xaFvWsS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.tail())"
      ],
      "metadata": {
        "id": "5unvAwReVNYC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "id": "6a6CVv1IvyN5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**6. Remove Stopwords**"
      ],
      "metadata": {
        "id": "cQzpUMq3b1PU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pip install nltk"
      ],
      "metadata": {
        "id": "E2C56KLOZcoS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "NOTE: BEFORE RUNNING MAKE SURE ALL THE THINGS AS PER EXPECTATION BECAUSE REMOVING STOPWORDS WILL TAKE TIME AROUND 2 HOURS AS PER DATA AROUND 28 LAKHS\n"
      ],
      "metadata": {
        "id": "NEqYoO73878r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "def remove_stopwords(text, language='english'):\n",
        "\n",
        "    nltk.download('stopwords')\n",
        "    nltk.download('punkt')\n",
        "\n",
        "    # Load the list of stopwords for the specified language\n",
        "    stop_words = set(stopwords.words(language))\n",
        "\n",
        "    # Tokenize the text into words\n",
        "    word_tokens = word_tokenize(text)\n",
        "\n",
        "    # Filter out the stopwords\n",
        "    filtered_text = [word for word in word_tokens if word.lower() not in stop_words]\n",
        "\n",
        "    # Join the filtered words back into a string\n",
        "    filtered_text = ' '.join(filtered_text)\n",
        "\n",
        "    return filtered_text\n"
      ],
      "metadata": {
        "id": "56Sou7sZZuOw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['text']=df['text'].apply(remove_stopwords)\n",
        "df"
      ],
      "metadata": {
        "id": "OBQJagjNaU0B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.head())\n",
        "df.shape"
      ],
      "metadata": {
        "id": "CQAs1cl4v13p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for creating the new dataset in desired directory\n",
        "out_dir = \"C:/ineuron/\"\n",
        "output_filename = \"Updated_text_data.csv\"\n",
        "\n",
        "output_filepath = os.path.join(out_dir + output_filename)\n",
        "\n",
        "if not os.path.exists(out_dir):\n",
        "  os.makedirs(out_dir)\n",
        "\n",
        "df.to_csv(output_filepath, index=False)\n",
        "print(f\"The new sheet dataset been upated to: {output_filepath}\")"
      ],
      "metadata": {
        "id": "lP0dbynDhAnu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**7. Expanding Abbreviations**"
      ],
      "metadata": {
        "id": "WZCIvahN5qf7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "# Fetch the slang dictionary from the provided URL\n",
        "slang_url = 'https://raw.githubusercontent.com/rishabhverma17/sms_slang_translator/master/slang.txt'\n",
        "slang_dict_response = requests.get(slang_url)\n",
        "\n",
        "# Check if the request was successful (status code 200)\n",
        "if slang_dict_response.status_code == 200:\n",
        "    # Convert slang text into a dictionary\n",
        "    abbr_dict = {}\n",
        "    for line in slang_dict_response.text.split('\\n'):\n",
        "        if '=' in line:\n",
        "            key, value = map(str.strip, line.split('='))\n",
        "            abbr_dict[key] = value\n",
        "else:\n",
        "    print(\"Failed to fetch slang dictionary. Status code:\", slang_dict_response.status_code)\n",
        "\n",
        "def expand_text(text):\n",
        "    words = text.split()\n",
        "    expanded_words = [abbr_dict.get(word.lower(), word) for word in words]\n",
        "    expanded_text = ' '.join(expanded_words)\n",
        "    return expanded_text\n"
      ],
      "metadata": {
        "id": "vVeqokNHkk8p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['text']=df['text'].apply(expand_text)\n",
        "df"
      ],
      "metadata": {
        "id": "960qxwEC5aC1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for creating the new dataset in desired directory\n",
        "out_dir = \"C:/ineuron/\"\n",
        "output_filename = \"Updated_text_data_abbre.csv\"\n",
        "\n",
        "output_filepath = os.path.join(out_dir + output_filename)\n",
        "\n",
        "if not os.path.exists(out_dir):\n",
        "  os.makedirs(out_dir)\n",
        "\n",
        "df.to_csv(output_filepath, index=False)\n",
        "print(f\"The new sheet dataset been upated to: {output_filepath}\")"
      ],
      "metadata": {
        "id": "684WnFyu6Ef6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**8. Remove Stemming**"
      ],
      "metadata": {
        "id": "kwhYY1lm7lbO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.stem import PorterStemmer\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "def stem_text(text):\n",
        "    # Initialize the Porter Stemmer\n",
        "    stemmer = PorterStemmer()\n",
        "\n",
        "    # Tokenize the text into individual words\n",
        "    tokens = word_tokenize(text)\n",
        "\n",
        "    # Stem each word in the text\n",
        "    stemmed_words = [stemmer.stem(word) for word in tokens]\n",
        "\n",
        "    # Join the stemmed words back into a string\n",
        "    stemmed_text = ' '.join(stemmed_words)\n",
        "\n",
        "    return stemmed_text"
      ],
      "metadata": {
        "id": "kP5dx1Z-6OxO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['text']=df['text'].apply(stem_text)\n",
        "df"
      ],
      "metadata": {
        "id": "a8zt7R8c7uKH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**9. Spelling correction**"
      ],
      "metadata": {
        "id": "MtlRvvcM75bC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from textblob import TextBlob\n",
        "def spell_check_text(text):\n",
        "    blob = TextBlob(text)\n",
        "    corrected_text = str(blob.correct())\n",
        "    return corrected_text\n",
        "\n",
        "df['text']=df['text'].apply(spell_check_text)\n",
        "df"
      ],
      "metadata": {
        "id": "NaS-a45U7-rd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**10. Manage Whitespace**"
      ],
      "metadata": {
        "id": "SAm4kl2u8evs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def whitespace_clean(text):\n",
        "    # Remove leading and trailing white spaces\n",
        "    clean_text = text.strip()\n",
        "\n",
        "    return clean_text\n",
        "\n",
        "df['text']=df['text'].apply(whitespace_clean)\n",
        "df"
      ],
      "metadata": {
        "id": "_lObVnMM8SjR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**11. Create Tokenize**"
      ],
      "metadata": {
        "id": "aLDQcF908meX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.tokenize import word_tokenize, sent_tokenize\n",
        "def tokenize_texts(input_series):\n",
        "    tokenized_data = input_series.apply(lambda text: (sent_tokenize(text), word_tokenize(text)))\n",
        "    return tokenized_data\n",
        "\n",
        "df['text']=tokenize_texts(df['text'])\n",
        "print(df[\"text\"])"
      ],
      "metadata": {
        "id": "F7vRPSv48qG4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for creating the last dataset in desired directory after all the preprocessing steps\n",
        "out_dir = \"C:/ineuron/\"\n",
        "output_filename = \"applying_all_preprocessing_steps.csv\"\n",
        "\n",
        "output_filepath = os.path.join(out_dir + output_filename)\n",
        "\n",
        "if not os.path.exists(out_dir):\n",
        "  os.makedirs(out_dir)\n",
        "\n",
        "df.to_csv(output_filepath, index=False)\n",
        "print(f\"The new sheet dataset been upated to: {output_filepath}\")"
      ],
      "metadata": {
        "id": "-GnDf2uIBfkM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv('/kaggle/working/applying_all_preprocessing_steps.csv')"
      ],
      "metadata": {
        "id": "GQ0DB9hnBW2r"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}