Data Engineering project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 505,
   "id": "b5722494",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 506,
   "id": "7c4fe92f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "import pytesseract\n",
    "import sys\n",
    "#Setting tesseract path in local\n",
    "pytesseract.pytesseract.tesseract_cmd = r'/opt/homebrew/Cellar/tesseract/5.1.0/bin/tesseract'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 507,
   "id": "11d9d564",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.common.keys import Keys #For accessing buttons\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.common.exceptions import StaleElementReferenceException\n",
    "import time\n",
    "import pandas as pd\n",
    "import math\n",
    "from urllib.request import urlopen,  Request\n",
    "\n",
    "#from PIL import Image, ImageOps, ImageEnhance\n",
    "#from io import StringIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 524,
   "id": "723f6a0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WDM] - \n",
      "\n",
      "[WDM] - ====== WebDriver manager ======\n",
      "[WDM] - Current google-chrome version is 99.0.4844\n",
      "[WDM] - Get LATEST driver version for 99.0.4844\n",
      "[WDM] - Driver [/Users/jeyasriramesh/.wdm/drivers/chromedriver/mac64/99.0.4844.51/chromedriver] found in cache\n"
     ]
    }
   ],
   "source": [
    "browser = webdriver.Chrome(ChromeDriverManager().install())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 509,
   "id": "b65b5339",
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_searchpage():\n",
    "    #Navigating to main frame inside homepage\n",
    "    home_url = \"https://www.sijilat.bh/internet_main.aspx\"\n",
    "    browser.get(home_url)\n",
    "    time.sleep(2)\n",
    "    browser.switch_to.frame(\"frame_main\")\n",
    "\n",
    "    #Closing pop-up\n",
    "    browser.switch_to.frame(0)\n",
    "    browser.find_element_by_xpath('//*[@id=\"bs_LblockPop\"]/div/div/a').click()\n",
    "\n",
    "    #Switch to the immediate parent frame\n",
    "    browser.switch_to.parent_frame()\n",
    "    \n",
    "    #Navigating to Target search page\n",
    "    browser.execute_script(\"fn_menu_click('01010000')\")\n",
    "    time.sleep(2)\n",
    "    #browser.switch_to.frame(\"frame_main\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 510,
   "id": "0825f387",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_captcha():\n",
    "    \n",
    "    captcha = ''\n",
    "    while len(captcha) < 4 : \n",
    "        \n",
    "        #Refresh search_page\n",
    "        to_searchpage()\n",
    "        \n",
    "        #Identify and download captcha\n",
    "        img_data = browser.find_element_by_id(\"BodyPlaceHolder_imgCaptcha\")\n",
    "        scrrenshot = img_data.screenshot_as_png\n",
    "        with open('VDfiles/captcha.png', 'wb') as f:\n",
    "            f.write(scrrenshot)\n",
    "    \n",
    "        #Extract captcha content\n",
    "        captcha = pytesseract.image_to_string('VDfiles/captcha.png') \n",
    "        captcha = captcha.replace(\" \", \"\").strip()\n",
    "    \n",
    "    return captcha\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 511,
   "id": "6f817636",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def check_captcha_text():\n",
    "    \n",
    "    #Check if any captcha alert is displayed\n",
    "    try:\n",
    "        if (browser.find_element_by_id(\"btnAlertOK\").is_displayed()):\n",
    "            browser.find_element_by_id(\"btnAlertOK\").click()\n",
    "            to_searchpage()\n",
    "            time.sleep(1)\n",
    "            captcha = get_captcha()\n",
    "            return captcha\n",
    "    except:\n",
    "        return '1'\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 523,
   "id": "84cfd566",
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = ['HSBC', 'SONY', 'QATAR','ALMAHBOOB','LESHBOONA','UNIVERSAL CATERING','INTERNATIONAL BOUTIQUE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "id": "559944b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "for company in companies:\n",
    "   \n",
    "    #Extracting Captcha from search page\n",
    "    captcha = get_captcha()\n",
    "    \n",
    "    #Commercial Name field id -  BodyPlaceHolder_txtCom\n",
    "    browser.find_element_by_id(\"BodyPlaceHolder_txtCom\").clear()\n",
    "    browser.find_element_by_id(\"BodyPlaceHolder_txtCom\").send_keys(company)\n",
    "\n",
    "    #Enter captcha\n",
    "    browser.find_element_by_id(\"BodyPlaceHolder_tb_captcha\").clear()\n",
    "    browser.find_element_by_id(\"BodyPlaceHolder_tb_captcha\").send_keys(captcha)\n",
    "    \n",
    "    #click search button\n",
    "    browser.find_element_by_id('BodyPlaceHolder_btnSearch').click()\n",
    "    time.sleep(3)\n",
    "    #Incase of captcha alert\n",
    "    if len(check_captcha_text()) > 1:\n",
    "        \n",
    "        captcha = check_captcha_text()\n",
    "        \n",
    "        #Enter captcha\n",
    "        browser.find_element_by_id(\"BodyPlaceHolder_tb_captcha\").clear()\n",
    "        browser.find_element_by_id(\"BodyPlaceHolder_tb_captcha\").send_keys(captcha)\n",
    "    \n",
    "        #Commercial Name field id -  BodyPlaceHolder_txtCom\n",
    "        browser.find_element_by_id(\"BodyPlaceHolder_txtCom\").clear()\n",
    "        browser.find_element_by_id(\"BodyPlaceHolder_txtCom\").send_keys(company)\n",
    "\n",
    "        #click search button\n",
    "        browser.find_element_by_id('BodyPlaceHolder_btnSearch').click()\n",
    "    \n",
    "    #Collecting records details in a table \n",
    "    time.sleep(3)\n",
    "    df_list = pd.read_html(browser.page_source) # this parses all the tables in webpages to a list\n",
    "    df = df_list[1]\n",
    "\n",
    "    #Getting 5 CR_NO records\n",
    "    for i in range(1):\n",
    "        cr_no = str(df.iloc[i][0]).split(\"-\")\n",
    "        \n",
    "        if i > 0:\n",
    "            browser.execute_script(\"window.history.go(-1)\")\n",
    "            time.sleep(2)\n",
    "        \n",
    "        #Navigating to company details by record\n",
    "        browser.execute_script(\"f_MoveLoading(this);f_CRDetailPage(\"+ cr_no[0] +\",\" + cr_no[1]+ \");\")\n",
    "        \n",
    "        #Saving screenshot\n",
    "        filename = company + \"_\" + str(cr_no[0])+ \"_\" + str(cr_no[1])\n",
    "        file_path = 'VDfiles/screenshot/'+filename+'.png'\n",
    "        \n",
    "        #Getting a screenshot\n",
    "        time.sleep(3)\n",
    "        browser.execute_script(\"document.body.style.zoom='50%'\") \n",
    "        time.sleep(1)\n",
    "        browser.find_element_by_tag_name('body').screenshot(file_path)\n",
    "        \n",
    "        #Enhancing screenshot\n",
    "\n",
    "        #box = (450, 220, 900, 600)\n",
    "        #im = Image.open(file_path)\n",
    "        #rgb_im = im.convert('RGB')\n",
    "        #region = rgb_im.crop(box)\n",
    "        #enhancer = ImageEnhance.Sharpness(im)\n",
    "        #enhanced_im = enhancer.enhance(1.0)\n",
    "        #r = ImageOps.invert(region.convert('L').point(lambda x: 255 if x > 254 else 0))\n",
    "        #enhanced_im.save('test.png',optimize=False, quality=99)\n",
    "        #region.save('screen.jpg', 'JPEG', optimize=False, quality=99)\n",
    "\n",
    "        #Converting company details into a dataframe\n",
    "        time.sleep(2)\n",
    "        \n",
    "        try: \n",
    "            #browser.switch_to.frame(\"frame_main\")\n",
    "            df_comp_list = pd.read_html(browser.page_source) # this parses all the tables in webpages to a list\n",
    "            df_comp = df_comp_list[1]\n",
    "            df_comp = df_comp[1:4]\n",
    "\n",
    "\n",
    "            #Formatting as per requirement\n",
    "            df_comp = df_comp.replace('Nationality', 'Country')\n",
    "\n",
    "            #print(df_comp)\n",
    "\n",
    "            #Converting contents to a dictionary (i.e key value pairs)\n",
    "            key_pair = [{i:j} for i, j in df_comp.iloc[:,[2,3]].values]\n",
    "            final_key_pair = key_pair[:len(key_pair)-2]\n",
    "            final_key_pair.extend([{i:j} for i, j in df_comp.iloc[:,[0,1]].values])\n",
    "            final_key_pair\n",
    "\n",
    "            #print(final_key_pair)\n",
    "\n",
    "            #writing the contents to a text file\n",
    "            filename_text = company + \"_\" + str(cr_no[0])+ \"_\" + str(cr_no[1])\n",
    "            file_path_text = 'VDfiles/keypairs_file/'+filename_text+'.txt'\n",
    "            with open(file_path_text, 'w') as f:\n",
    "                print(final_key_pair, file=f)\n",
    "        except:\n",
    "            pass\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9ac8f38",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5402ad8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f60428d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}