Assignment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: selenium in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (4.14.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.26 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from urllib3[socks]<3,>=1.26->selenium) (1.26.16)\n",
      "Requirement already satisfied: trio~=0.17 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from selenium) (0.22.2)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from selenium) (0.11.1)\n",
      "Requirement already satisfied: certifi>=2021.10.8 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from selenium) (2023.7.22)\n",
      "Requirement already satisfied: attrs>=20.1.0 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio~=0.17->selenium) (23.1.0)\n",
      "Requirement already satisfied: sortedcontainers in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: idna in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio~=0.17->selenium) (3.4)\n",
      "Requirement already satisfied: outcome in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio~=0.17->selenium) (1.3.0)\n",
      "Requirement already satisfied: sniffio in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio~=0.17->selenium) (1.3.0)\n",
      "Requirement already satisfied: cffi>=1.14 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio~=0.17->selenium) (1.15.1)\n",
      "Requirement already satisfied: wsproto>=0.14 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from trio-websocket~=0.9->selenium) (1.2.0)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: pycparser in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.21)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.14.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (1.24.3)\n",
      "Requirement already satisfied: pandas in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (2.0.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas) (2023.3)\n",
      "Requirement already satisfied: numpy>=1.21.0 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas) (1.24.3)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n",
      "Requirement already satisfied: regex in c:\\users\\vinay\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (2023.8.8)\n"
     ]
    }
   ],
   "source": [
    "!pip install selenium\n",
    "!pip install numpy\n",
    "!pip install pandas\n",
    "!pip install regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from selenium import webdriver \n",
    "from selenium.webdriver.common.by import By\n",
    "import time\n",
    "import regex as re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()\n",
    "driver.get(\"https://qcpi.questcdn.com/cdn/posting/?group=1950787&provider=1950787\")\n",
    "time.sleep(3)\n",
    "\n",
    "#To find the first a element which contains the Questnumber\n",
    "a = driver.find_element(By.XPATH,'/html/body/div[2]/div[2]/div[2]/div/div[3]/div[6]/div/div/div/div[3]/div/table/tbody/tr[1]/td[2]/a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "#List of elements\n",
    "quest_no = []\n",
    "Est_Notes = []\n",
    "Description = []\n",
    "Closing_date = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to scrap the details\n",
    "def Scrap(n):\n",
    "    a.click()\n",
    "    for i in range(n):\n",
    "        time.sleep(3)\n",
    "        quest_no.append(''.join(re.findall(r'\\d', driver.find_element(By.XPATH,'/html/body/div[2]/div[2]/div[2]/div/div[4]/div[1]/div[2]/div[2]/div/h4/span/b').text)))\n",
    "        Est_Notes.append(driver.find_element(By.XPATH,\"/html/body/div[2]/div[2]/div[2]/div/div[4]/div[1]/div[4]/div[2]/div/div/div[2]/div/table/tbody/tr[3]/td[2]\").text)\n",
    "        Closing_date.append(driver.find_element(By.XPATH,\"/html/body/div[2]/div[2]/div[2]/div/div[4]/div[1]/div[4]/div[2]/div/div/div[2]/div/table/tbody/tr[1]/td[2]\").text)\n",
    "        Description.append(driver.find_element(By.XPATH,\"/html/body/div[2]/div[2]/div[2]/div/div[4]/div[1]/div[4]/div[2]/div/div/div[3]/div/table/tbody/tr[3]/td[2]\").text)\n",
    "        time.sleep(3)\n",
    "        driver.find_element(By.ID,'id_prevnext_next').click()\n",
    "        time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to convert the list of data into dataframe\n",
    "def df(quest_no,Est_Notes,Description,Closing_date):\n",
    "    Data = pd.DataFrame(list(zip(quest_no,Est_Notes,Description,Closing_date)),columns=['Quest_No','Est_notes','Description','Closing_date'])\n",
    "    return Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Scrap(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}