PhantomTroupe_HtmlProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bb50a902",
   "metadata": {},
   "source": [
    "# USING HTML FILES"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ab5eadb",
   "metadata": {},
   "source": [
    "### Failed Attempt\n",
    "Reference - https://towardsdatascience.com/website-data-cleaning-in-python-for-nlp-dda282a7a871"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9015c26c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn.preprocessing as preprocessing \n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import sklearn.metrics as metrics \n",
    "import sklearn.svm as svm \n",
    "import sklearn.naive_bayes as naive_bayes\n",
    "from sklearn.linear_model import LogisticRegression \n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.naive_bayes import GaussianNB, BernoulliNB\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "import seaborn as sns \n",
    "import json\n",
    "import nltk\n",
    "import string\n",
    "import re\n",
    "import os\n",
    "from boilerpy3 import extractors \n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1105714f",
   "metadata": {},
   "source": [
    "## Using boilerpy3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "290e3d12",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_html(html_path):\n",
    "    # Text extraction with boilerpy3\n",
    "    html_extractor = extractors.ArticleExtractor()\n",
    "    return condense_newline(html_extractor.get_content_from_file(html_path))\n",
    "\n",
    "def remove_newline(text):\n",
    "    return '\\n'.join([p for p in re.split('\\n|\\r', text) if len(p) > 0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c254baf5",
   "metadata": {},
   "source": [
    "## Using BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1f4f6ee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_html(html_path):\n",
    "    with open(html_path, 'r') as fr:\n",
    "        html_content = fr.read()\n",
    "        soup = BeautifulSoup(html_content, 'html.parser')\n",
    "\n",
    "        if not soup.find():\n",
    "            raise ValueError(\"File is not a valid HTML file\")\n",
    "\n",
    "        tag_meta_language = soup.head.find(\"meta\", attrs={\"http-equiv\": \"content-language\"})\n",
    "        if tag_meta_language:\n",
    "            document_language = tag_meta_language[\"content\"]\n",
    "            if document_language and document_language not in [\"en\", \"en-us\", \"en-US\"]:\n",
    "                raise ValueError(\"Language {} is not english\".format(document_language))\n",
    "                \n",
    "        TAGS = ['p','q,'t']\n",
    "        return ' '.join([remove_newline(tag.text) for tag in soup.findAll(TAGS)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "12677de8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def html_to_text(folder):\n",
    "    parsed_texts = []\n",
    "    filepaths = os.listdir(folder)\n",
    "\n",
    "    for filepath in filepaths:\n",
    "        filepath_full = os.path.join(folder, filepath)\n",
    "#         print(filepath_full)\n",
    "        parsed_texts.append(parse_html(filepath_full))\n",
    "    return parsed_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a3d28d2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html_content/0\n",
      "1\n",
      "html_content/1\n",
      "1\n",
      "html_content/10\n",
      "1\n"
     ]
    },
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'charmap' codec can't decode byte 0x8f in position 43589: character maps to <undefined>",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13024/375232164.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mdirectory\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'html_content/'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mparsed_texts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhtml_to_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdirectory\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13024/3347828275.py\u001b[0m in \u001b[0;36mhtml_to_text\u001b[1;34m(folder)\u001b[0m\n\u001b[0;32m      8\u001b[0m         \u001b[1;31m#if filepath_full.endswith(\".html\"):\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m         \u001b[0mparsed_texts\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparse_html\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_full\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mparsed_texts\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_13024/423079518.py\u001b[0m in \u001b[0;36mparse_html\u001b[1;34m(html_path)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mparse_html\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhtml_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhtml_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'r'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mfr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m         \u001b[0mhtml_content\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m         \u001b[0msoup\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhtml_content\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'html.parser'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Python\\Python39\\lib\\encodings\\cp1252.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcharmap_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdecoding_table\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mStreamWriter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCodec\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStreamWriter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnicodeDecodeError\u001b[0m: 'charmap' codec can't decode byte 0x8f in position 43589: character maps to <undefined>"
     ]
    }
   ],
   "source": [
    "directory = 'html_content/'\n",
    "parsed_texts = html_to_text(directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ac39fd1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65bb4919",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}