Web_Scrapping_Internship_Final_Module (2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nRVIMRIInphM",
    "outputId": "0c137327-6890-440f-f3b4-580aee741647"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: selenium in d:\\ashwa\\ana\\lib\\site-packages (4.1.3)\n",
      "Requirement already satisfied: trio~=0.17 in d:\\ashwa\\ana\\lib\\site-packages (from selenium) (0.20.0)\n",
      "Requirement already satisfied: urllib3[secure,socks]~=1.26 in d:\\ashwa\\ana\\lib\\site-packages (from selenium) (1.26.7)\n",
      "Requirement already satisfied: trio-websocket~=0.9 in d:\\ashwa\\ana\\lib\\site-packages (from selenium) (0.9.2)\n",
      "Requirement already satisfied: async-generator>=1.9 in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (1.10)\n",
      "Requirement already satisfied: sniffio in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (1.2.0)\n",
      "Requirement already satisfied: idna in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (3.2)\n",
      "Requirement already satisfied: attrs>=19.2.0 in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (21.2.0)\n",
      "Requirement already satisfied: outcome in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (1.1.0)\n",
      "Requirement already satisfied: sortedcontainers in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (2.4.0)\n",
      "Requirement already satisfied: cffi>=1.14 in d:\\ashwa\\ana\\lib\\site-packages (from trio~=0.17->selenium) (1.14.6)\n",
      "Requirement already satisfied: pycparser in d:\\ashwa\\ana\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.20)\n",
      "Requirement already satisfied: wsproto>=0.14 in d:\\ashwa\\ana\\lib\\site-packages (from trio-websocket~=0.9->selenium) (1.1.0)\n",
      "Requirement already satisfied: PySocks!=1.5.7,<2.0,>=1.5.6 in d:\\ashwa\\ana\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (1.7.1)\n",
      "Requirement already satisfied: pyOpenSSL>=0.14 in d:\\ashwa\\ana\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (21.0.0)\n",
      "Requirement already satisfied: cryptography>=1.3.4 in d:\\ashwa\\ana\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (3.4.8)\n",
      "Requirement already satisfied: certifi in d:\\ashwa\\ana\\lib\\site-packages (from urllib3[secure,socks]~=1.26->selenium) (2021.10.8)\n",
      "Requirement already satisfied: six>=1.5.2 in d:\\ashwa\\ana\\lib\\site-packages (from pyOpenSSL>=0.14->urllib3[secure,socks]~=1.26->selenium) (1.16.0)\n",
      "Requirement already satisfied: h11<1,>=0.9.0 in d:\\ashwa\\ana\\lib\\site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.13.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TlVqmaLBlvgT",
    "outputId": "e8197de1-223b-4b70-fbf3-75d609af6557"
   },
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "from selenium.webdriver.support.select import Select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "path =  'D:\\chromedriver.exe'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/910283902.py:1: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  wd = webdriver.Chrome(path)\n"
     ]
    }
   ],
   "source": [
    "wd = webdriver.Chrome(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "iIQTKNSOl8kq"
   },
   "outputs": [],
   "source": [
    "wd.get('https://www.giiresearch.com/material_report.shtml')#for getting data from website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "B4xwClsMmB5G"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df=pd.DataFrame(columns=[\"Title\",\"Published By\",\"Product Code\",\"Pubilshed Date\",\"Content Info\",\"Price\"],index=[x for x in range(125)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "BId0zGwImFkh"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    Title Published By Product Code Pubilshed Date Content Info Price\n",
      "0     NaN          NaN          NaN            NaN          NaN   NaN\n",
      "1     NaN          NaN          NaN            NaN          NaN   NaN\n",
      "2     NaN          NaN          NaN            NaN          NaN   NaN\n",
      "3     NaN          NaN          NaN            NaN          NaN   NaN\n",
      "4     NaN          NaN          NaN            NaN          NaN   NaN\n",
      "..    ...          ...          ...            ...          ...   ...\n",
      "120   NaN          NaN          NaN            NaN          NaN   NaN\n",
      "121   NaN          NaN          NaN            NaN          NaN   NaN\n",
      "122   NaN          NaN          NaN            NaN          NaN   NaN\n",
      "123   NaN          NaN          NaN            NaN          NaN   NaN\n",
      "124   NaN          NaN          NaN            NaN          NaN   NaN\n",
      "\n",
      "[125 rows x 6 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df) #check for validation of data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "ItNXHSx1WuHf"
   },
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "XGi9LdgEpJLw"
   },
   "outputs": [],
   "source": [
    "#lists for appending data from multiple pages\n",
    "list_Title=[]\n",
    "published_By=[]\n",
    "product_Code=[]\n",
    "published_Date=[]\n",
    "content_Info=[]\n",
    "price=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jsMuqtsXnaCp",
    "outputId": "c2287d70-cbcf-4313-f788-37225c3f25e9"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:3: DeprecationWarning: find_elements_by_xpath is deprecated. Please use find_elements(by=By.XPATH, value=xpath) instead\n",
      "  listTitle=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//td[@class=\"plist_title\"]')\n",
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:5: DeprecationWarning: find_elements_by_xpath is deprecated. Please use find_elements(by=By.XPATH, value=xpath) instead\n",
      "  publishedBy=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_pubinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:7: DeprecationWarning: find_elements_by_xpath is deprecated. Please use find_elements(by=By.XPATH, value=xpath) instead\n",
      "  productCode=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_codeinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:9: DeprecationWarning: find_elements_by_xpath is deprecated. Please use find_elements(by=By.XPATH, value=xpath) instead\n",
      "  publishedDate=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_dateinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:11: DeprecationWarning: find_elements_by_xpath is deprecated. Please use find_elements(by=By.XPATH, value=xpath) instead\n",
      "  contentInfo=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_pageinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:13: DeprecationWarning: find_elements_by_xpath is deprecated. Please use find_elements(by=By.XPATH, value=xpath) instead\n",
      "  priceProduct=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_priceinfo\"]//div[@class=\"plist_info_dd\"]')\n",
      "C:\\Users\\ashwa\\AppData\\Local\\Temp/ipykernel_16724/3693007703.py:22: DeprecationWarning: find_element_by_xpath is deprecated. Please use find_element(by=By.XPATH, value=xpath) instead\n",
      "  next=wd.find_element_by_xpath('//div[@class=\"wrapper\"]//div[@id=\"Content_Body\"]//div[@class=\"page_btn\"]//button[@class=\"btn_next\"]')\n"
     ]
    }
   ],
   "source": [
    "for k in range(5):\n",
    "  #title\n",
    "  listTitle=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//td[@class=\"plist_title\"]')\n",
    "  #Published by\n",
    "  publishedBy=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_pubinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
    "  #Product Code\n",
    "  productCode=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_codeinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
    "  #Pubilshed Date\n",
    "  publishedDate=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_dateinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
    "  #content info\n",
    "  contentInfo=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_pageinfo\"]//div[@class=\"plist_info_dd2\"]')\n",
    "  #price\n",
    "  priceProduct=wd.find_elements_by_xpath('//*[@id=\"Content_Body\"]/form[1]//table[@class=\"plist_item\"]//div[@class=\"plist_priceinfo\"]//div[@class=\"plist_info_dd\"]')\n",
    "  time.sleep(5)\n",
    "  for i in range(len(listTitle)):\n",
    "    list_Title.append(listTitle[i].text)\n",
    "    published_By.append(publishedBy[i].text)\n",
    "    product_Code.append(productCode[i].text)\n",
    "    published_Date.append(publishedDate[i].text)\n",
    "    content_Info.append(contentInfo[i].text)\n",
    "    price.append(priceProduct[i].text)\n",
    "  next=wd.find_element_by_xpath('//div[@class=\"wrapper\"]//div[@id=\"Content_Body\"]//div[@class=\"page_btn\"]//button[@class=\"btn_next\"]')\n",
    "  next.click()\n",
    "  time.sleep(5)\n",
    "    \n",
    "#dear sir,\n",
    "#we can extract data of all pages using the for loop, we just need increase the range from 5 to 1023 and it will scrape all the data from the website.\n",
    "#since it was taking much time in my pc so i did not do."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "BDjbPX-updhK"
   },
   "outputs": [],
   "source": [
    "for u in range(len(list_Title)):\n",
    "  df[\"Title\"][u]=list_Title[u]\n",
    "  df[\"Published By\"][u]=published_By[u]\n",
    "  df[\"Product Code\"][u]=product_Code[u]\n",
    "  df[\"Pubilshed Date\"][u]=published_Date[u]\n",
    "  df[\"Content Info\"][u]=content_Info[u]\n",
    "  df[\"Price\"][u]=price[u].replace(\"\\nAdd to Compare\",\"\")\n",
    "  #print(list_Title[u]+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V422f6hxqSkd",
    "outputId": "fe359ae4-0e42-46c9-d0f7-52b2f2407b03"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                 Title  \\\n",
      "0    Global Nano-Copper Particles Market Profession...   \n",
      "1    Global Monocrystalline Diamond Market Report, ...   \n",
      "2    Global Industrial Inkjet Printheads Market Rep...   \n",
      "3       Global Float Glass Market Research Report 2022   \n",
      "4    Global Electrostatic Chucks (ESCs) In Semicond...   \n",
      "..                                                 ...   \n",
      "120       Glass Fiber Industry Forecasts - China Focus   \n",
      "121  Automotive Fluids & Chemicals Industry Forecas...   \n",
      "122  Cement, Concrete & Aggregate Industry Forecast...   \n",
      "123  Industrial Rubber Products Industry Forecasts ...   \n",
      "124    Coated Fabrics Industry Forecasts - China Focus   \n",
      "\n",
      "                                  Published By Product Code  Pubilshed Date  \\\n",
      "0                                   QYResearch      1070481  April 20, 2022   \n",
      "1                                   QYResearch      1070480  April 20, 2022   \n",
      "2                                   QYResearch      1070478  April 20, 2022   \n",
      "3                                   QYResearch      1070476  April 20, 2022   \n",
      "4                                   QYResearch      1070474  April 20, 2022   \n",
      "..                                         ...          ...             ...   \n",
      "120  Asia Market Information & Development Co.       339271   April 5, 2022   \n",
      "121  Asia Market Information & Development Co.       339269   April 5, 2022   \n",
      "122  Asia Market Information & Development Co.       339268   April 5, 2022   \n",
      "123  Asia Market Information & Development Co.       339265   April 5, 2022   \n",
      "124  Asia Market Information & Development Co.       339264   April 5, 2022   \n",
      "\n",
      "    Content Info     Price  \n",
      "0      103 Pages  USD 3500  \n",
      "1       96 Pages  USD 3350  \n",
      "2      102 Pages  USD 3350  \n",
      "3      151 Pages  USD 2900  \n",
      "4      135 Pages  USD 2900  \n",
      "..           ...       ...  \n",
      "120     84 Pages  USD 1800  \n",
      "121     89 Pages  USD 1800  \n",
      "122     97 Pages  USD 1800  \n",
      "123    113 Pages  USD 1800  \n",
      "124     97 Pages  USD 1800  \n",
      "\n",
      "[125 rows x 6 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "kla-QB5koH03"
   },
   "outputs": [],
   "source": [
    "df.to_csv(\"scrapped_data_internshir.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Web_Scrapping_Internship_Final_Module.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}