glassdoor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests #the library to make requests on the web (i.e, to look for things!)\n",
    "import re\n",
    "import pandas as pd\n",
    "from pprint import pprint  #to print (a bit more) prettily your request output.\n",
    "from bs4 import BeautifulSoup #the library to process the output of requests. See below.\n",
    "from datetime import datetime, timedelta\n",
    "from os import path\n",
    "import urllib3\n",
    "import numpy as np\n",
    "import os\n",
    "import pdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def master(key):\n",
    "    df = do_get_job_info(key)\n",
    "    df = clean_df(df)\n",
    "    save_csv(df)\n",
    "    return(\"scraped\")\n",
    "\n",
    "def do_get_job_info(key):\n",
    "    jobContainer = get_jobContainer(key)\n",
    "    jobInfos = list(map(get_job_info, jobContainer['jobContainer'], np.repeat(jobContainer['session'], len(jobContainer['jobContainer']))))  \n",
    "    df = pd.DataFrame(jobInfos)\n",
    "    return(df)\n",
    "\n",
    "def get_jobContainer(key):\n",
    "    session = requests.Session()\n",
    "    session.post(glassdoor()['url'] + \"/profile/login_input.htm\", data=glassdoor()['credentials'])\n",
    "    s = session.get(browse(key), headers={'User-Agent':'Mozilla/5.0'})\n",
    "    s = BeautifulSoup(s.text, 'html.parser') \n",
    "    s = s.find(id='JobResults')\n",
    "    jobContainer = s.find_all(class_='jobContainer')\n",
    "    return({'jobContainer':jobContainer, 'session':session})\n",
    "\n",
    "def glassdoor():\n",
    "    u = 'https://www.glassdoor.com'\n",
    "    c = {'_username':'[e.tenezakis@gmail.com]', '_password':'[aimilios1908]'}\n",
    "    return({'url':u,'credentials':c})\n",
    "\n",
    "def browse(key):\n",
    "    req  = glassdoor()['url'] + '/Job/jobs.htm?suggestCount=0&suggestChosen=false&clickSource=searchBtn&typedKeyword='+ \\\n",
    "    key + '&sc.keyword='+ key + '&locT=C&locId=3021489&jobType='\n",
    "    return(req)\n",
    "\n",
    "def get_job_info(job, session):\n",
    "    id = job.find('span')['data-job-id']\n",
    "    days_ago = job.find('div', class_='jobLabels')\n",
    "    job_url = job.find('a')['href']\n",
    "    url = glassdoor()['url'] + job_url \n",
    "    employer = job.find('div', class_='jobInfoItem jobEmpolyerName')\n",
    "    location1 = 'Seoul'\n",
    "    location2 = job.find('span', class_='subtle loc')\n",
    "    jobdesc = 'JobDesc' + id\n",
    "    detail = session.get(url, headers={'User-Agent':'Mozilla/5.0'})\n",
    "    detail = BeautifulSoup(detail.text, 'html.parser') \n",
    "    title = detail.find('title')\n",
    "    description = detail.find(id='JobDescriptionContainer')\n",
    "    if None in (id, url, employer,location1,location2, title,description):\n",
    "        jobinfo = None\n",
    "    else:\n",
    "        jobinfo = {'title':title.text, \\\n",
    "           'id':id,\\\n",
    "           'url':url,\\\n",
    "           'employer':employer.text.lstrip(),\\\n",
    "           'location1':location1,\\\n",
    "           'location2':location2.text,\\\n",
    "           'description':description.text,\\\n",
    "           'days_ago':days_ago.text}\n",
    "    return(jobinfo)\n",
    "\n",
    "\n",
    "def clean_df(df):\n",
    "    df['title'] = df['title'].str.replace('|'.join(['/ ', ' in ','_','\\| Glassdoor','Job']),'') \n",
    "    df['title'] = df.apply(lambda row: row['title'].replace(row['employer'],''), axis=1)\n",
    "    df['title'] = df.apply(lambda row: row['title'].replace(row['location1'],''), axis=1)\n",
    "    df['title'] = df.apply(lambda row: row['title'].replace(row['location2'],''), axis=1)    \n",
    "    df['title'] = df['title'].str.strip()\n",
    "    df['description'] = df.apply(lambda row: row['description'][row['description'].rindex('}')+1:], axis=1)\n",
    "    df['description'] = df['description'].str.strip()\n",
    "    df['id'] = df['id'].str.strip()\n",
    "    df['id'] = df['id'].astype(int)\n",
    "    df['days_ago'] = df.apply(lambda row: int(re.findall(r'\\d+', row['days_ago'])[0]), axis=1)\n",
    "    df['published_on'] = df.apply(lambda row: (datetime.now()-timedelta(row['days_ago'])).strftime(\"%d/%m/%Y\"), axis=1)\n",
    "    df['scraped_on'] = datetime.now().strftime(\"%d/%m/%Y\")\n",
    "    del df['days_ago']\n",
    "    return(df)\n",
    "\n",
    "def dupl_remover(df):\n",
    "    df = df.sort_values(['id', 'scraped_on']) \n",
    "    unique = df.drop_duplicates(subset ='id', keep = 'first')\n",
    "    return(unique)\n",
    "\n",
    "def save_csv(new):\n",
    "    if os.path.exists(path()):\n",
    "        old = open_csv()\n",
    "        new = old.append(new, ignore_index=True)\n",
    "        new = dupl_remover(new)\n",
    "    new[list(new)] = new[list(new)].astype(str)\n",
    "    new.to_csv(path(), index=False)\n",
    "\n",
    "def path():\n",
    "    p = '/Users/emile/Documents/web_scraping/glassdoor_jobs.csv'\n",
    "    return(p)\n",
    "\n",
    "def open_csv():\n",
    "    csv = pd.read_csv(path())\n",
    "    return(csv)\n",
    "\n",
    "def hp(h):\n",
    "  print(h.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'scraped'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "master('Data Science')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}