mission_to_mars.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape():\n",
    "    # Dependencies\n",
    "    from bs4 import BeautifulSoup\n",
    "    import requests\n",
    "    from splinter import Browser\n",
    "    import pandas as pd\n",
    "\n",
    "    #setup resulting dict\n",
    "    results = {}\n",
    "\n",
    "    #--------------------------------------------\n",
    "    # SCRAPE LATEST NEWS\n",
    "    #--------------------------------------------\n",
    "\n",
    "    # https://splinter.readthedocs.io/en/latest/drivers/chrome.html\n",
    "    !which chromedriver\n",
    "\n",
    "    # have to run browser object\n",
    "    executable_path = {'executable_path': '/usr/local/bin/chromedriver'}\n",
    "    browser = Browser('chrome', **executable_path, headless=False)\n",
    "\n",
    "    # URL of NASA's mars news\n",
    "    url = 'https://mars.nasa.gov/news/?page=0&per_page=40&order=publish_date+desc%2Ccreated_at+desc&search=&category=19%2C165%2C184%2C204&blank_scope=Latest'\n",
    "\n",
    "    # launch browser\n",
    "    browser.visit(url)\n",
    "\n",
    "    # create beautifulsoup object\n",
    "    soup = BeautifulSoup(browser.html, 'html.parser')\n",
    "\n",
    "    # find latest news date, title and body (first appearanse of a tag in html)\n",
    "    news_date = soup.find('div', class_='list_date').text\n",
    "    news_title = soup.find('div', class_='content_title').text\n",
    "    news_p = soup.find('div', class_='article_teaser_body').text\n",
    "\n",
    "    # save to results\n",
    "    results['news_date']  = news_date\n",
    "    results['news_title'] = news_title\n",
    "    results['news_p']     = news_p\n",
    "\n",
    "\n",
    "    #--------------------------------------------\n",
    "    # SCRAPE FEATURED IMAGE\n",
    "    #--------------------------------------------\n",
    "\n",
    "    # URL of mars images page\n",
    "    url = 'https://www.jpl.nasa.gov/'\n",
    "    url1 = 'spaceimages/?search=&category=Mars'\n",
    "\n",
    "    # launch browser\n",
    "    browser.visit(url+url1)\n",
    "\n",
    "    # create beautifulsoup object\n",
    "    soup = BeautifulSoup(browser.html, 'html.parser')\n",
    "\n",
    "    # remove last '/' symbol from url\n",
    "    # locate the 'style' attribute of 'article' tag\n",
    "    # split by quotes and access an element of resulting list\n",
    "    # combine the two text elements get the image url\n",
    "    featured_image_url = url[:-1] + soup.find('article', class_='carousel_item')['style'].split(\"'\")[1]\n",
    "    results['featured_image_url'] = featured_image_url\n",
    "\n",
    "\n",
    "    #--------------------------------------------\n",
    "    # SCRAPE MARS'S WEATHER\n",
    "    #--------------------------------------------\n",
    "\n",
    "    # twitter url to visit\n",
    "    url = 'https://twitter.com/marswxreport?lang=en'\n",
    "\n",
    "    # launch browser\n",
    "    browser.visit(url)\n",
    "\n",
    "    # create beautifulsoup object\n",
    "    soup = BeautifulSoup(browser.html, 'html.parser')\n",
    "\n",
    "    # localate the first tweet and extract text from it\n",
    "    mars_weather = soup.find('div', class_='js-tweet-text-container').text.split('\\n')[1]\n",
    "    results['mars_weather'] = mars_weather\n",
    "\n",
    "\n",
    "    #--------------------------------------------\n",
    "    # SCRAPE MARS'S FACTS\n",
    "    #--------------------------------------------\n",
    "\n",
    "    # url to visit\n",
    "    url = 'http://space-facts.com/mars/'\n",
    "\n",
    "    # use pandas to extract table from the page\n",
    "    facts_table = pd.read_html(url)\n",
    "\n",
    "    # create df with columns' headers\n",
    "    df = facts_table[0]\n",
    "    df.columns = ['Parameter','Value']\n",
    "\n",
    "    # convert to html table\n",
    "    facts_table = df.to_html()\n",
    "\n",
    "    results['facts_table'] = facts_table\n",
    "\n",
    "\n",
    "    #--------------------------------------------\n",
    "    # SCRAPE HEMISPHERES' IMAGES\n",
    "    #--------------------------------------------\n",
    "\n",
    "    # url to visit\n",
    "    url = 'https://astrogeology.usgs.gov'\n",
    "    url1 = '/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'\n",
    "\n",
    "    # launch browser\n",
    "    browser.visit(url+url1)\n",
    "\n",
    "    # create beautifulsoup object\n",
    "    soup = BeautifulSoup(browser.html, 'html.parser')\n",
    "\n",
    "    # find section with images on the page\n",
    "    div = soup.find('div', class_='results').findAll('div', class_='description')\n",
    "\n",
    "    # extract urls\n",
    "    urls = []\n",
    "    for item in div:\n",
    "        links = item.findAll('a')\n",
    "        for a in links:\n",
    "            urls.append(a.get('href'))\n",
    "\n",
    "    # visit urls to extract links and titles\n",
    "    hemisphere_image_urls = []\n",
    "    for u in urls:\n",
    "        result_dict = {}\n",
    "        browser.visit(url+u)\n",
    "        soup = BeautifulSoup(browser.html, 'html.parser')\n",
    "\n",
    "        # find and extract links to images\n",
    "        links = soup.find('div', class_='downloads').findAll('a')\n",
    "        result_dict['img_url_jpeg'] = links[0].get('href')\n",
    "        result_dict['img_url'] = links[1].get('href')\n",
    "\n",
    "        # extract title\n",
    "        result_dict['title'] = soup.find('h2', class_='title').text\n",
    "\n",
    "        # append dict to results list\n",
    "        hemisphere_image_urls.append(result_dict)\n",
    "\n",
    "    results['hemisphere_image_urls'] = hemisphere_image_urls\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}