Data_Extraction_and_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A6jujqieacpE",
        "outputId": "0c53f1cb-fca8-4c96-86be-406c7a43e5e5"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import necessary libraries\n",
        "import os\n",
        "import requests\n",
        "import pandas as pd\n",
        "from bs4 import BeautifulSoup\n",
        "from textblob import TextBlob\n",
        "from nltk.tokenize import sent_tokenize, word_tokenize\n",
        "import nltk\n",
        "\n",
        "# Ensure required NLTK data packages are downloaded\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "\n",
        "# Define utility functions\n",
        "def load_words(file_path, encoding='utf-8'):\n",
        "    try:\n",
        "        with open(file_path, 'r', encoding=encoding) as file:\n",
        "            words = set(word.strip() for word in file.readlines())\n",
        "            print(f\"Loaded {len(words)} words from {file_path}\")\n",
        "            return words\n",
        "    except Exception as e:\n",
        "        print(f\"Error reading {file_path}: {e}\")\n",
        "        return set()\n",
        "\n",
        "def load_specific_stopwords(folder_path, filenames, encoding='utf-8'):\n",
        "    stopwords = set()\n",
        "    try:\n",
        "        for filename in filenames:\n",
        "            file_path = os.path.join(folder_path, filename)\n",
        "            stopwords.update(load_words(file_path, encoding))\n",
        "        print(f\"Loaded {len(stopwords)} stopwords from {folder_path}\")\n",
        "    except Exception as e:\n",
        "        print(f\"Error reading stopwords from {folder_path}: {e}\")\n",
        "    return stopwords\n",
        "\n",
        "def save_text_to_file(text, file_path):\n",
        "    with open(file_path, 'w') as file:\n",
        "        file.write(text)\n",
        "\n",
        "def extract_article_content(url):\n",
        "    response = requests.get(url)\n",
        "    soup = BeautifulSoup(response.content, 'html.parser')\n",
        "    title = soup.find('h1').text if soup.find('h1') else ''\n",
        "    paragraphs = soup.find_all('p')\n",
        "    article_text = ' '.join([para.text for para in paragraphs])\n",
        "    return title, article_text\n",
        "\n",
        "def count_syllables(word):\n",
        "    vowels = \"aeiouy\"\n",
        "    word = word.lower().strip()\n",
        "    count = 0\n",
        "    if word and word[0] in vowels:\n",
        "        count += 1\n",
        "    for index in range(1, len(word)):\n",
        "        if word[index] in vowels and word[index - 1] not in vowels:\n",
        "            count += 1\n",
        "    if word.endswith(\"e\"):\n",
        "        count -= 1\n",
        "    if count == 0:\n",
        "        count += 1\n",
        "    return count\n",
        "\n",
        "def analyze_text(text, positive_words, negative_words, stopwords):\n",
        "    blob = TextBlob(text)\n",
        "    words = [word for word in blob.words if word.lower() not in stopwords]\n",
        "    positive_score = sum(1 for word in words if word in positive_words)\n",
        "    negative_score = sum(1 for word in words if word in negative_words)\n",
        "    polarity_score = blob.sentiment.polarity\n",
        "    subjectivity_score = blob.sentiment.subjectivity\n",
        "    sentences = sent_tokenize(text)\n",
        "    complex_words = [word for word in words if len(word) > 2]\n",
        "\n",
        "    avg_sentence_length = len(words) / len(sentences) if sentences else 0\n",
        "    percentage_of_complex_words = len(complex_words) / len(words) if words else 0\n",
        "    fog_index = 0.4 * (avg_sentence_length + percentage_of_complex_words)\n",
        "    avg_number_of_words_per_sentence = avg_sentence_length\n",
        "    complex_word_count = len(complex_words)\n",
        "    word_count = len(words)\n",
        "    syllables_per_word = sum(count_syllables(word) for word in words) / len(words) if words else 0\n",
        "    personal_pronouns = sum(1 for word in words if word.lower() in [\"i\", \"we\", \"my\", \"ours\", \"us\"])\n",
        "    avg_word_length = sum(len(word) for word in words) / len(words) if words else 0\n",
        "\n",
        "    return {\n",
        "        'POSITIVE SCORE': positive_score,\n",
        "        'NEGATIVE SCORE': negative_score,\n",
        "        'POLARITY SCORE': polarity_score,\n",
        "        'SUBJECTIVITY SCORE': subjectivity_score,\n",
        "        'AVG SENTENCE LENGTH': avg_sentence_length,\n",
        "        'PERCENTAGE OF COMPLEX WORDS': percentage_of_complex_words,\n",
        "        'FOG INDEX': fog_index,\n",
        "        'AVG NUMBER OF WORDS PER SENTENCE': avg_number_of_words_per_sentence,\n",
        "        'COMPLEX WORD COUNT': complex_word_count,\n",
        "        'WORD COUNT': word_count,\n",
        "        'SYLLABLE PER WORD': syllables_per_word,\n",
        "        'PERSONAL PRONOUNS': personal_pronouns,\n",
        "        'AVG WORD LENGTH': avg_word_length\n",
        "    }\n",
        "\n",
        "# Load input data\n",
        "input_file_path = '/content/drive/MyDrive/Data Science Internship-Blackcoffer/Input.xlsx'  # Update the path as needed\n",
        "input_df = pd.read_excel(input_file_path)\n",
        "\n",
        "# Load positive and negative word dictionaries from the master dictionary folder\n",
        "master_dictionary_folder_path = '/content/drive/MyDrive/Data Science Internship-Blackcoffer/MasterDictionary'  # Update the path as needed\n",
        "positive_words_path = os.path.join(master_dictionary_folder_path, 'positive-words.txt')\n",
        "negative_words_path = os.path.join(master_dictionary_folder_path, 'negative-words.txt')\n",
        "positive_words = load_words(positive_words_path, encoding='latin1')\n",
        "negative_words = load_words(negative_words_path, encoding='latin1')\n",
        "\n",
        "# Load specific stopwords from the specified folder\n",
        "stopwords_folder_path = '/content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords'  # Update the path as needed\n",
        "stopword_filenames = [\n",
        "    'StopWords_Currencies.txt',\n",
        "    'StopWords_Auditor.txt',\n",
        "    'StopWords_DatesandNumbers.txt',\n",
        "    'StopWords_Generic.txt',\n",
        "    'StopWords_GenericLong.txt',\n",
        "    'StopWords_Geographic.txt',\n",
        "    'StopWords_Names.txt'\n",
        "]  # List stopword filenames here\n",
        "stopwords = load_specific_stopwords(stopwords_folder_path, stopword_filenames, encoding='latin1')\n",
        "\n",
        "# Extract articles\n",
        "articles = []\n",
        "for _, row in input_df.iterrows():\n",
        "    url_id = row['URL_ID']\n",
        "    url = row['URL']\n",
        "    title, text = extract_article_content(url)\n",
        "    articles.append((url_id, url, title, text))\n",
        "\n",
        "# Save extracted articles\n",
        "articles_df = pd.DataFrame(articles, columns=['URL_ID', 'URL', 'Title', 'Text'])\n",
        "articles_df.to_csv('extracted_articles.csv', index=False)\n",
        "\n",
        "# Analyze extracted articles\n",
        "analysis_results = []\n",
        "for _, row in articles_df.iterrows():\n",
        "    analysis = analyze_text(row['Text'], positive_words, negative_words, stopwords)\n",
        "    analysis['URL_ID'] = row['URL_ID']\n",
        "    analysis['URL'] = row['URL']\n",
        "    analysis_results.append(analysis)\n",
        "\n",
        "# Save analysis results\n",
        "analysis_df = pd.DataFrame(analysis_results)\n",
        "analysis_df = analysis_df[['URL_ID', 'URL', 'POSITIVE SCORE', 'NEGATIVE SCORE', 'POLARITY SCORE', 'SUBJECTIVITY SCORE',\n",
        "                           'AVG SENTENCE LENGTH', 'PERCENTAGE OF COMPLEX WORDS', 'FOG INDEX', 'AVG NUMBER OF WORDS PER SENTENCE',\n",
        "                           'COMPLEX WORD COUNT', 'WORD COUNT', 'SYLLABLE PER WORD', 'PERSONAL PRONOUNS', 'AVG WORD LENGTH']]\n",
        "analysis_df.to_excel('analysis_results.xlsx', index=False)\n",
        "\n",
        "# Display the first few rows of the analysis results\n",
        "analysis_df.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 625
        },
        "id": "7qp_BGEZQ9eW",
        "outputId": "e86e687a-9707-43eb-e99f-fddb9094179d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loaded 2006 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/MasterDictionary/positive-words.txt\n",
            "Loaded 4783 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/MasterDictionary/negative-words.txt\n",
            "Loaded 85 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_Currencies.txt\n",
            "Loaded 8 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_Auditor.txt\n",
            "Loaded 108 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_DatesandNumbers.txt\n",
            "Loaded 121 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_Generic.txt\n",
            "Loaded 570 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_GenericLong.txt\n",
            "Loaded 195 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_Geographic.txt\n",
            "Loaded 11897 words from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords/StopWords_Names.txt\n",
            "Loaded 12919 stopwords from /content/drive/MyDrive/Data Science Internship-Blackcoffer/StopWords\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       URL_ID                                                URL  \\\n",
              "0  bctech2011  https://insights.blackcoffer.com/ml-and-ai-bas...   \n",
              "1  bctech2012  https://insights.blackcoffer.com/streamlined-i...   \n",
              "2  bctech2013  https://insights.blackcoffer.com/efficient-dat...   \n",
              "3  bctech2014  https://insights.blackcoffer.com/effective-man...   \n",
              "4  bctech2015  https://insights.blackcoffer.com/streamlined-t...   \n",
              "\n",
              "   POSITIVE SCORE  NEGATIVE SCORE  POLARITY SCORE  SUBJECTIVITY SCORE  \\\n",
              "0              34               8        0.069339            0.386899   \n",
              "1               4               0        0.059315            0.331447   \n",
              "2               4               0        0.059315            0.331447   \n",
              "3               4               0        0.059315            0.331447   \n",
              "4               4               0        0.059315            0.331447   \n",
              "\n",
              "   AVG SENTENCE LENGTH  PERCENTAGE OF COMPLEX WORDS  FOG INDEX  \\\n",
              "0            26.333333                     0.963608  10.918776   \n",
              "1            28.142857                     0.959391  11.640899   \n",
              "2            27.714286                     0.958763  11.469219   \n",
              "3            28.000000                     0.959184  11.583673   \n",
              "4            28.142857                     0.959391  11.640899   \n",
              "\n",
              "   AVG NUMBER OF WORDS PER SENTENCE  COMPLEX WORD COUNT  WORD COUNT  \\\n",
              "0                         26.333333                 609         632   \n",
              "1                         28.142857                 189         197   \n",
              "2                         27.714286                 186         194   \n",
              "3                         28.000000                 188         196   \n",
              "4                         28.142857                 189         197   \n",
              "\n",
              "   SYLLABLE PER WORD  PERSONAL PRONOUNS  AVG WORD LENGTH  \n",
              "0           2.528481                  0         7.488924  \n",
              "1           2.441624                  0         7.502538  \n",
              "2           2.443299                  0         7.500000  \n",
              "3           2.443878                  0         7.500000  \n",
              "4           2.446701                  0         7.517766  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a9d7704f-366d-4b72-849f-4ef51bbe9866\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URL_ID</th>\n",
              "      <th>URL</th>\n",
              "      <th>POSITIVE SCORE</th>\n",
              "      <th>NEGATIVE SCORE</th>\n",
              "      <th>POLARITY SCORE</th>\n",
              "      <th>SUBJECTIVITY SCORE</th>\n",
              "      <th>AVG SENTENCE LENGTH</th>\n",
              "      <th>PERCENTAGE OF COMPLEX WORDS</th>\n",
              "      <th>FOG INDEX</th>\n",
              "      <th>AVG NUMBER OF WORDS PER SENTENCE</th>\n",
              "      <th>COMPLEX WORD COUNT</th>\n",
              "      <th>WORD COUNT</th>\n",
              "      <th>SYLLABLE PER WORD</th>\n",
              "      <th>PERSONAL PRONOUNS</th>\n",
              "      <th>AVG WORD LENGTH</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>bctech2011</td>\n",
              "      <td>https://insights.blackcoffer.com/ml-and-ai-bas...</td>\n",
              "      <td>34</td>\n",
              "      <td>8</td>\n",
              "      <td>0.069339</td>\n",
              "      <td>0.386899</td>\n",
              "      <td>26.333333</td>\n",
              "      <td>0.963608</td>\n",
              "      <td>10.918776</td>\n",
              "      <td>26.333333</td>\n",
              "      <td>609</td>\n",
              "      <td>632</td>\n",
              "      <td>2.528481</td>\n",
              "      <td>0</td>\n",
              "      <td>7.488924</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>bctech2012</td>\n",
              "      <td>https://insights.blackcoffer.com/streamlined-i...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0.059315</td>\n",
              "      <td>0.331447</td>\n",
              "      <td>28.142857</td>\n",
              "      <td>0.959391</td>\n",
              "      <td>11.640899</td>\n",
              "      <td>28.142857</td>\n",
              "      <td>189</td>\n",
              "      <td>197</td>\n",
              "      <td>2.441624</td>\n",
              "      <td>0</td>\n",
              "      <td>7.502538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>bctech2013</td>\n",
              "      <td>https://insights.blackcoffer.com/efficient-dat...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0.059315</td>\n",
              "      <td>0.331447</td>\n",
              "      <td>27.714286</td>\n",
              "      <td>0.958763</td>\n",
              "      <td>11.469219</td>\n",
              "      <td>27.714286</td>\n",
              "      <td>186</td>\n",
              "      <td>194</td>\n",
              "      <td>2.443299</td>\n",
              "      <td>0</td>\n",
              "      <td>7.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>bctech2014</td>\n",
              "      <td>https://insights.blackcoffer.com/effective-man...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0.059315</td>\n",
              "      <td>0.331447</td>\n",
              "      <td>28.000000</td>\n",
              "      <td>0.959184</td>\n",
              "      <td>11.583673</td>\n",
              "      <td>28.000000</td>\n",
              "      <td>188</td>\n",
              "      <td>196</td>\n",
              "      <td>2.443878</td>\n",
              "      <td>0</td>\n",
              "      <td>7.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>bctech2015</td>\n",
              "      <td>https://insights.blackcoffer.com/streamlined-t...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0.059315</td>\n",
              "      <td>0.331447</td>\n",
              "      <td>28.142857</td>\n",
              "      <td>0.959391</td>\n",
              "      <td>11.640899</td>\n",
              "      <td>28.142857</td>\n",
              "      <td>189</td>\n",
              "      <td>197</td>\n",
              "      <td>2.446701</td>\n",
              "      <td>0</td>\n",
              "      <td>7.517766</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a9d7704f-366d-4b72-849f-4ef51bbe9866')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-a9d7704f-366d-4b72-849f-4ef51bbe9866 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-a9d7704f-366d-4b72-849f-4ef51bbe9866');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-d78b055c-a6ef-47c4-9362-d1bd28281879\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-d78b055c-a6ef-47c4-9362-d1bd28281879')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-d78b055c-a6ef-47c4-9362-d1bd28281879 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "analysis_df",
              "summary": "{\n  \"name\": \"analysis_df\",\n  \"rows\": 147,\n  \"fields\": [\n    {\n      \"column\": \"URL_ID\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 147,\n        \"samples\": [\n          \"bctech2136\",\n          \"bctech2062\",\n          \"bctech2149\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"URL\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 147,\n        \"samples\": [\n          \"https://insights.blackcoffer.com/lipsync-automation-for-celebrities-and-influencers/\",\n          \"https://insights.blackcoffer.com/an-etl-solution-for-an-internet-publishing-firm/\",\n          \"https://insights.blackcoffer.com/data-etl-local-service-ads-leads-to-bigquery/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"POSITIVE SCORE\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6,\n        \"min\": 3,\n        \"max\": 34,\n        \"num_unique_values\": 24,\n        \"samples\": [\n          26,\n          9,\n          34\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"NEGATIVE SCORE\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2,\n        \"min\": 0,\n        \"max\": 14,\n        \"num_unique_values\": 12,\n        \"samples\": [\n          6,\n          14,\n          8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"POLARITY SCORE\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.04784095371313584,\n        \"min\": -0.11211484593837534,\n        \"max\": 0.2557551707551708,\n        \"num_unique_values\": 134,\n        \"samples\": [\n          0.08191710871288338,\n          0.054232171337434486,\n          0.11728795895462561\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SUBJECTIVITY SCORE\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.06486905495297854,\n        \"min\": 0.2442241968557758,\n        \"max\": 0.5958858543417368,\n        \"num_unique_values\": 139,\n        \"samples\": [\n          0.26748838317645657,\n          0.3828151646333464,\n          0.38217189314750294\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AVG SENTENCE LENGTH\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.110261879986631,\n        \"min\": 11.129032258064516,\n        \"max\": 37.285714285714285,\n        \"num_unique_values\": 133,\n        \"samples\": [\n          20.27777777777778,\n          32.5,\n          17.294117647058822\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"PERCENTAGE OF COMPLEX WORDS\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.014884105071501143,\n        \"min\": 0.8947368421052632,\n        \"max\": 0.9822784810126582,\n        \"num_unique_values\": 131,\n        \"samples\": [\n          0.9653465346534653,\n          0.9777070063694268,\n          0.9776674937965261\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"FOG INDEX\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.443755783091155,\n        \"min\": 4.840018700327256,\n        \"max\": 15.305090311986863,\n        \"num_unique_values\": 143,\n        \"samples\": [\n          13.385641025641025,\n          6.361437367888981,\n          7.3583138173302105\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AVG NUMBER OF WORDS PER SENTENCE\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.110261879986631,\n        \"min\": 11.129032258064516,\n        \"max\": 37.285714285714285,\n        \"num_unique_values\": 133,\n        \"samples\": [\n          20.27777777777778,\n          32.5,\n          17.294117647058822\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"COMPLEX WORD COUNT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 116,\n        \"min\": 146,\n        \"max\": 917,\n        \"num_unique_values\": 112,\n        \"samples\": [\n          335,\n          321,\n          192\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"WORD COUNT\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 123,\n        \"min\": 153,\n        \"max\": 1007,\n        \"num_unique_values\": 117,\n        \"samples\": [\n          204,\n          200,\n          353\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SYLLABLE PER WORD\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.1041678439925858,\n        \"min\": 2.1145510835913313,\n        \"max\": 2.859550561797753,\n        \"num_unique_values\": 140,\n        \"samples\": [\n          2.236111111111111,\n          2.2857142857142856,\n          2.1317567567567566\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"PERSONAL PRONOUNS\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AVG WORD LENGTH\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.35746102069400904,\n        \"min\": 6.38390092879257,\n        \"max\": 9.235955056179776,\n        \"num_unique_values\": 143,\n        \"samples\": [\n          9.235955056179776\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    }
  ]
}