programming/beautiful_soup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web scraping and parsing with Beautiful Soup & Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "b'<!DOCTYPE html>\\n<html>\\n    <head>\\n        <title>A simple example page</title>\\n    </head>\\n    <body>\\n        <p>Here is some simple content for this page.</p>\\n    </body>\\n</html>'\n"
     ]
    }
   ],
   "source": [
    "page = requests.get(\"http://dataquestio.github.io/web-scraping-pages/simple.html\")\n",
    "print(page.status_code)\n",
    "print(page.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   A simple example page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <p>\n",
      "   Here is some simple content for this page.\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "soup = BeautifulSoup(page.content, 'html.parser')\n",
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['html', '\\n', <html>\n",
       " <head>\n",
       " <title>A simple example page</title>\n",
       " </head>\n",
       " <body>\n",
       " <p>Here is some simple content for this page.</p>\n",
       " </body>\n",
       " </html>]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(soup.children)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      " <head>\n",
      "  <title>\n",
      "   A simple example page\n",
      "  </title>\n",
      " </head>\n",
      " <body>\n",
      "  <p>\n",
      "   Here is some simple content for this page.\n",
      "  </p>\n",
      " </body>\n",
      "</html>\n"
     ]
    }
   ],
   "source": [
    "print(soup.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html <class 'bs4.element.Doctype'>\n",
      "\n",
      " <class 'bs4.element.NavigableString'>\n",
      "<html>\n",
      "<head>\n",
      "<title>A simple example page</title>\n",
      "</head>\n",
      "<body>\n",
      "<p>Here is some simple content for this page.</p>\n",
      "</body>\n",
      "</html> <class 'bs4.element.Tag'>\n"
     ]
    }
   ],
   "source": [
    "for item in soup.children:\n",
    "    print(item, type(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = list(soup.children)[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n', <head>\n",
       " <title>A simple example page</title>\n",
       " </head>, '\\n', <body>\n",
       " <p>Here is some simple content for this page.</p>\n",
       " </body>, '\\n']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(html.children)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\n', <p>Here is some simple content for this page.</p>, '\\n']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "body = list(html.children)[3]\n",
    "list(body.children)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p>Here is some simple content for this page.</p>\n",
      "Here is some simple content for this page.\n"
     ]
    }
   ],
   "source": [
    "p = list(body.children)[1]\n",
    "print(p)\n",
    "print(p.get_text())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding Tag Instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p>Here is some simple content for this page.</p>]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p>Here is some simple content for this page.</p>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find('p')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classes/IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<html>\n",
       "<head>\n",
       "<title>A simple example page</title>\n",
       "</head>\n",
       "<body>\n",
       "<div>\n",
       "<p class=\"inner-text first-item\" id=\"first\">\n",
       "                First paragraph.\n",
       "            </p>\n",
       "<p class=\"inner-text\">\n",
       "                Second paragraph.\n",
       "            </p>\n",
       "</div>\n",
       "<p class=\"outer-text first-item\" id=\"second\">\n",
       "<b>\n",
       "                First outer paragraph.\n",
       "            </b>\n",
       "</p>\n",
       "<p class=\"outer-text\">\n",
       "<b>\n",
       "                Second outer paragraph.\n",
       "            </b>\n",
       "</p>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page = requests.get(\"http://dataquestio.github.io/web-scraping-pages/ids_and_classes.html\")\n",
    "soup = BeautifulSoup(page.content, 'html.parser')\n",
    "soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"outer-text first-item\" id=\"second\">\n",
       " <b>\n",
       "                 First outer paragraph.\n",
       "             </b>\n",
       " </p>, <p class=\"outer-text\">\n",
       " <b>\n",
       "                 Second outer paragraph.\n",
       "             </b>\n",
       " </p>]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('p', class_='outer-text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"outer-text first-item\" id=\"second\">\n",
       " <b>\n",
       "                 First outer paragraph.\n",
       "             </b>\n",
       " </p>, <p class=\"outer-text\">\n",
       " <b>\n",
       "                 Second outer paragraph.\n",
       "             </b>\n",
       " </p>]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(class_=\"outer-text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"inner-text first-item\" id=\"first\">\n",
       "                 First paragraph.\n",
       "             </p>]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(id=\"first\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Search with CSS Selectors\n",
    "Find all p tags within a div:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"inner-text first-item\" id=\"first\">\n",
       "                 First paragraph.\n",
       "             </p>, <p class=\"inner-text\">\n",
       "                 Second paragraph.\n",
       "             </p>]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select(\"div p\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}