webscraper_echo_updated.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from datetime import datetime, timedelta, date\n",
    "import hashlib\n",
    "import requests\n",
    "import os\n",
    "import sys\n",
    "from os.path import *\n",
    "import urllib\n",
    "import xlrd\n",
    "sys.path.insert(0, '../')\n",
    "try:\n",
    "    from bs4 import (\n",
    "        BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString,\n",
    "        Declaration, Doctype)\n",
    "    _DECLARATION_OR_DOCTYPE = (Declaration, Doctype)\n",
    "except ImportError:\n",
    "    from BeautifulSoup import (\n",
    "        BeautifulSoup, Tag, Comment, ProcessingInstruction, NavigableString,\n",
    "        Declaration)\n",
    "    _DECLARATION_OR_DOCTYPE = Declaration\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import Select\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as cond\n",
    "from selenium.common.exceptions import NoAlertPresentException\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "from pyvirtualdisplay.display import Display\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import Select\n",
    "import time #using this library to pause or \"sleep\" program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.support import expected_conditions as EC "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Steps (broad overview)\n",
    "#1. login to gmail (use skytruth email)\n",
    "#2. open correct email (Echo Notify - need to make an account)\n",
    "#3. Click on link in email \n",
    "#4. Scrape detailed facility report (still working on this)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tatiana\\AppData\\Local\\Temp\\ipykernel_25124\\2321693607.py:2: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver=webdriver.Firefox(executable_path=f'C:\\Drivers\\FirefoxDriver\\geckodriver.exe')\n"
     ]
    }
   ],
   "source": [
    "#Open gmail - set up driver, navigate to website \n",
    "driver=webdriver.Firefox(executable_path=f'C:\\Drivers\\FirefoxDriver\\geckodriver.exe')\n",
    "driver.implicitly_wait(60) # web driver waits 60 seconds before throwing a NoSuchElementException - applies to every action - only have to write this once. \n",
    "driver.maximize_window()\n",
    "#driver.get('https://mail.google.com')\n",
    "\n",
    "driver.get('https://echo.epa.gov/detailed-facility-report?fid=CABAA00006013A0010')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Login to gmail - enter username, press next, enter password"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#enter username\n",
    "driver.find_element(By.ID, \"identifierId\").send_keys(\"tatiana@skytruth.org\") #enter your username \n",
    "#click the 'next' button\n",
    "driver.find_element(By.ID, \"identifierNext\").click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#enter password\n",
    "driver.find_element(By.NAME, \"Passwd\").send_keys(\"#######\") #enter your password\n",
    "#click the 'next' button \n",
    "driver.find_element(By.ID, \"passwordNext\").click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#may need to manually enter a verification code and press \"next\" button to complete login."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#need to add a \"wait\" here, because usually get sent an email about a security alert, and then the following code has errors executing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Email clicked: EPA ECHO Notify: Air Results for SEP 24, 2022 - OCT 01, 2022\n"
     ]
    }
   ],
   "source": [
    "#should also maybe make this into a function.\n",
    "#Will eventually need to edit this to comb through  multiple pages if necesssary, if desired email is not on first page.\n",
    "email_titles=driver.find_elements(By.XPATH, \"//span[@class='bog']\") #create list; all email subjects are of class 'bog'\n",
    "for title in email_titles: #now comb through list; if email subject matches our desired email subject, click the email.\n",
    "    if title.text == \"EPA ECHO Notify: Air Results for SEP 24, 2022 - OCT 01, 2022\": #if the subject in the list matches our desired email subject...\n",
    "        subject=title.text\n",
    "        title.click()\n",
    "        print(\"Email clicked:\", subject) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "wait = WebDriverWait(driver, 10) #explicit wait - used to tell the Web Driver to wait for certain conditions (Expected Conditions) or maximum time exceeded before throwing “ElementNotVisibleException” exception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_tab_email = driver.current_window_handle #store id of the original tab (the email tab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#click on link\n",
    "#Notes: The <a> tag defines a hyperlink, which is used to link from one page to another.\n",
    "#The most important attribute of the <a> element is the href attribute, which indicates the link's destination.\n",
    "\n",
    "#will need to find a way to automate this process rather than having to manually enter the link text - we'll figure that out later. \n",
    "#will also need to deal with a list of links. \n",
    "driver.find_element(By.LINK_TEXT,'CABAA00006013A0010').click()\n",
    "\n",
    "# Wait for the new window or tab (selenium views windows and tabs as being the same)\n",
    "wait.until(EC.number_of_windows_to_be(2))\n",
    "# Loop through until we find a new window handle\n",
    "for window_handle in driver.window_handles:\n",
    "    if window_handle != original_tab_email:\n",
    "        driver.switch_to.window(window_handle)\n",
    "        break\n",
    "wait.until(EC.presence_of_element_located((By.TAG_NAME, 'h1')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CHEVRON REFINERY RICHMOND\n",
      "841 CHEVRON WAY, RICHMOND, CA 94801\n",
      "FRS ID:  110020506460\n",
      "EPA Region:  09\n",
      "Latitude:  37.938779\n",
      "Longitude:  -122.396453\n",
      "Locational Data Source:  EIS\n",
      "Industries:  Merchant Wholesalers, Nondurable Goods\n",
      "Indian country:  N\n"
     ]
    }
   ],
   "source": [
    "#facility info \n",
    "\n",
    "#can't put this one in the loop because the xpath is different.\n",
    "facility_address=driver.find_element(By.XPATH, \"//section[@class='echo-subsection echo-grid-two-thirds']/div/div\").text\n",
    "print(facility_address)\n",
    "\n",
    "facility_info_row_labels=[\"FRS ID: \",\"EPA Region: \", \"Latitude: \", \"Longitude: \", \"Locational Data Source: \", \"Industries: \", \"Indian country: \"]\n",
    "\n",
    "for i in range(1, 8, 1): #(initialization, stop (noninclusive), increment)\n",
    "    facility_info_row=driver.find_element(By.XPATH, \"//section[@class='echo-subsection echo-grid-two-thirds']/div/dl/div[\"+str(i)+\"]/dd\").text\n",
    "    facility_info_row_label=facility_info_row_labels[i-1]\n",
    "    print(facility_info_row_label, facility_info_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scrape first two rows (row heading and violation info underneath) for the high priority violation. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n"
     ]
    }
   ],
   "source": [
    "#it appears that there are always 11 columns, but just in case, i'll count them up before i start scraping info. \n",
    "columns=driver.find_elements(By.XPATH, \"//table[@id='enforcementAndComplianceSummary']/thead/tr/th\")\n",
    "max_columns=len(columns)+1 #add 1 - since there are 11 columns, we need 12 for our noninclusive stop in the loop.\n",
    "print(max_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statute: CAA\n",
      "Compliance Monitoring Activities (5 years): 14\n",
      "Date of Last Compliance Monitoring Activity: 08/02/2022\n",
      "Compliance Status: High Priority Violation\n",
      "Qtrs with NC (of 12): 12\n",
      "Qtrs with Significant Violation: 12\n",
      "Informal Enforcement Actions (5 years): 340\n",
      "Formal Enforcement Actions (5 years): 169\n",
      "Penalties from Formal Enforcement Actions (5 years): $685,500\n",
      "EPA Cases (5 years): 1\n",
      "Penalties from EPA Cases (5 years): $1,930,300\n"
     ]
    }
   ],
   "source": [
    "for i in range(1, max_columns, 1): #(initialization, stop (noninclusive), increment)\n",
    "    header_label=driver.find_element(By.XPATH, \"//table[@id='enforcementAndComplianceSummary']/thead/tr/th[\"+str(i)+\"]\").text #finds text inside the column header cell \n",
    "    cell=driver.find_element(By.XPATH, \"//table[@id='enforcementAndComplianceSummary']/tbody/tr/td[\"+str(i)+\"]\").text #finds text inside the row cell \n",
    "    print(header_label, \": \", cell, sep = '') #sep='' avoids unnecessary whitespaces between header_label and colon."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ed8e88c9092603f5c604b52f358e31b84ef56c994a59054b7891a45d175e344d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}