news.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##東森新聞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests \n",
    "from bs4 import BeautifulSoup\n",
    "import cookielib\n",
    "import json\n",
    "import time \n",
    "\n",
    "\n",
    "urls_dic = {\"綜合\":\"http://news.ebc.net.tw/apps/inc/newsList_General.inc\",\n",
    "            \"社會\":\"http://news.ebc.net.tw/apps/inc/newsList_social.inc\",\n",
    "            \"生活\":\"http://news.ebc.net.tw/apps/inc/newsList_Lifestyle.inc\",\n",
    "            \"娛樂\":\"http://news.ebc.net.tw/apps/inc/newsList_Entertainment.inc\",\n",
    "            \"國際\":\"http://news.ebc.net.tw/apps/inc/newsList_Politics.inc\",\n",
    "            \"最多人瀏覽\":\"http://news.ebc.net.tw/apps/inc/newsList_MostClick.inc\",\n",
    "            \"正妹\":\"http://news.ebc.net.tw/apps/inc/newsList_PrettyGirls.inc\",\n",
    "            \"財經\":\"http://news.ebc.net.tw/apps/inc/newsList_Finance.inc\",\n",
    "            \"兩岸\":\"http://news.ebc.net.tw/apps/inc/newsList_China.inc\",\n",
    "            \"政治\":\"http://news.ebc.net.tw/apps/inc/newsList_Politics.inc\"}\n",
    "\n",
    "jar = cookielib.CookieJar()\n",
    "headers = {'Host':'news.ebc.net.tw',\n",
    "'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:38.0) Gecko/20100101 Firefox/38.0',\n",
    "'Accept':'text/html, */*; q=0.01',\n",
    "'Accept-Language':'zh-TW,zh;q=0.8,en-US;q=0.5,en;q=0.3',\n",
    "'Accept-Encoding':'gzip, deflate',\n",
    "'X-Requested-With':'XMLHttpRequest',\n",
    "'Referer':'http://news.ebc.net.tw/apps/newsList.htm?menu=4',\n",
    "'Cookie': jar,\n",
    "'Connection':'keep-alive'}\n",
    "\n",
    "# all news\n",
    "L = []\n",
    "for url in urls_dic:\n",
    "    res = requests.get(urls_dic[url],headers = headers)\n",
    "    res.encoding=\"utf-8\"\n",
    "    soup = BeautifulSoup(res.text)\n",
    "    for block in soup.select(\".newslist-box\"):\n",
    "        title=block.select('.newslist-title-txt')[0].text\n",
    "        time=block.select('.gray_text')[0].text\n",
    "        time= time.split(\"|\")[0]\n",
    "        aLink='http://news.ebc.net.tw/apps/' + block.select('a')[0]['href']\n",
    "        L.append({'Category':url,'Title':title,'Link':aLink,'Time':time})\n",
    "# pandas\n",
    "import pandas as pd\n",
    "df = pd.DataFrame(L)\n",
    "df\n",
    "\n",
    "# write in csv\n",
    "f=open(\"ETTV.csv\",\"w\")\n",
    "df.to_csv(\"ETTV.csv\",encoding=\"utf-8\")\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'split'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-71-d8fa40365367>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mlinks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Link'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mlink\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/inc'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/inc'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mlinks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'Link'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlink\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'split'"
     ]
    }
   ],
   "source": [
    "# content in all Link:\n",
    "res = requests.get(url)\n",
    "soup = BeautifulSoup(res.text)\n",
    "soup.select(\"#divContent\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 東森新聞雲"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import cookielib\n",
    "import math\n",
    "import re\n",
    "from datetime import date,datetime, timedelta \n",
    "import time\n",
    "\n",
    "# 各類新聞連結\n",
    "def ETToday_links(days,sleep_time=0):\n",
    "    #day = str(date.today()- timedelta(1))\n",
    "\n",
    "    ds = []\n",
    "    for i in xrange(1,days+1):\n",
    "        ds.append(str(date.today()- timedelta(i)))\n",
    "    \n",
    "    for day in ds:\n",
    "        urls_dic = {\"政治\":\"http://www.ettoday.net/news/news-list-%s-1.htm\"%day,\n",
    "                    \"財經\":\"http://www.ettoday.net/news/news-list-%s-17.htm\"%day,\n",
    "                    \"國際\":\"http://www.ettoday.net/news/news-list-%s-2.htm\"%day,\n",
    "                    \"大陸\":\"http://www.ettoday.net/news/news-list-%s-3.htm\"%day,\n",
    "                    \"社會\":\"http://www.ettoday.net/news/news-list-%s-6.htm\"%day,\n",
    "                    \"地方\":\"http://www.ettoday.net/news/news-list-%s-7.htm\"%day,\n",
    "                    \"影劇\":\"http://www.ettoday.net/news/news-list-%s-9.htm\"%day,\n",
    "                    \"3C\":\"http://www.ettoday.net/news/news-list-%s-20.htm\"%day\n",
    "                   }\n",
    "\n",
    "        L = []\n",
    "        for url in urls_dic:\n",
    "            res = requests.get(urls_dic[url])\n",
    "            res.encoding=\"utf-8\"\n",
    "            soup = BeautifulSoup(res.text)\n",
    "            page = re.search('\\((\\d+).\\)',soup.select(\".info\")[0].text).group(1)\n",
    "            pageCount = int(math.ceil(float(page)/30))\n",
    "            for block in soup.select(\"#all-news-list > h3\"):\n",
    "                title=block.select('a')[0].text\n",
    "                aLink=block.select('a')[0]['href']\n",
    "                L.append({'Category':block.select('em')[0].text,'Title':title,'Link':aLink,'Time':day})\n",
    "                time.sleep(sleep_time)\n",
    "            \n",
    "            if pageCount > 1:\n",
    "                for i in xrange(2,pageCount+1):\n",
    "                    url_1=str(urls_dic[url]).split(\".htm\")[0] + \"-\" + str(i) + \".htm\"\n",
    "                    res = requests.get(url_1)\n",
    "                    res.encoding=\"utf-8\"\n",
    "                    soup = BeautifulSoup(res.text)\n",
    "                    for block in soup.select(\"#all-news-list > h3\"):\n",
    "                        title=block.select('a')[0].text\n",
    "                        aLink=block.select('a')[0]['href']\n",
    "                        L.append({'Category':block.select('em')[0].text,'Title':title,'Link':aLink,'Time':day})\n",
    "                        time.sleep(sleep_time)\n",
    "            \n",
    "    # pandas\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(L)\n",
    "\n",
    "    # write in csv\n",
    "    f=open(\"ETToday_links.csv\",\"w\")\n",
    "    df.to_csv(\"ETToday_links.csv\",encoding=\"utf-8\")\n",
    "    f.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#各類新聞連結指定日期\n",
    "\n",
    "def ETToday_links_day(day,sleep_time=0):\n",
    "    urls_dic = {\"政治\":\"http://www.ettoday.net/news/news-list-%s-1.htm\"%day,\n",
    "                \"財經\":\"http://www.ettoday.net/news/news-list-%s-17.htm\"%day,\n",
    "                \"國際\":\"http://www.ettoday.net/news/news-list-%s-2.htm\"%day,\n",
    "                \"大陸\":\"http://www.ettoday.net/news/news-list-%s-3.htm\"%day,\n",
    "                \"社會\":\"http://www.ettoday.net/news/news-list-%s-6.htm\"%day,\n",
    "                \"地方\":\"http://www.ettoday.net/news/news-list-%s-7.htm\"%day,\n",
    "                \"影劇\":\"http://www.ettoday.net/news/news-list-%s-9.htm\"%day,\n",
    "                \"3C\":\"http://www.ettoday.net/news/news-list-%s-20.htm\"%day\n",
    "                       }\n",
    "    L = []\n",
    "    for url in urls_dic:\n",
    "        res = requests.get(urls_dic[url])\n",
    "        res.encoding=\"utf-8\"\n",
    "        soup = BeautifulSoup(res.text)\n",
    "        page = re.search('\\((\\d+).\\)',soup.select(\".info\")[0].text).group(1)\n",
    "        pageCount = int(math.ceil(float(page)/30))\n",
    "        for block in soup.select(\"#all-news-list > h3\"):\n",
    "            title=block.select('a')[0].text\n",
    "            aLink=block.select('a')[0]['href']\n",
    "            L.append({'Category':block.select('em')[0].text,'Title':title,'Link':aLink,'Time':day})\n",
    "            time.sleep(sleep_time)\n",
    "\n",
    "        if pageCount > 1:\n",
    "            for i in xrange(2,pageCount+1):\n",
    "                url_1=str(urls_dic[url]).split(\".htm\")[0] + \"-\" + str(i) + \".htm\"\n",
    "                res = requests.get(url_1)\n",
    "                res.encoding=\"utf-8\"\n",
    "                soup = BeautifulSoup(res.text)\n",
    "                for block_1 in soup.select(\"#all-news-list > h3\"):\n",
    "                    title=block_1.select('a')[0].text\n",
    "                    aLink=block_1.select('a')[0]['href']\n",
    "                    L.append({'Category':block_1.select('em')[0].text,'Title':title,'Link':aLink,'Time':day})\n",
    "                    time.sleep(sleep_time)\n",
    "    # pandas\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(L)\n",
    "\n",
    "    # write in csv\n",
    "    f=open(\"ETToday_links.csv\",\"w\")\n",
    "    df.to_csv(\"ETToday_links\"+\"_day\"+\".csv\",encoding=\"utf-8\")\n",
    "    f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "《東森新聞雲》掌握新媒體快速崛起　兩岸產學界驚豔\n"
     ]
    }
   ],
   "source": [
    "# 各新聞內容\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "url = \"http://www.ettoday.net/news/20150613/520214.htm\"\n",
    "res=requests.get(url)\n",
    "soup = BeautifulSoup(res.text)\n",
    "#print soup.select('.title.clearfix')[0].text\n",
    "\n",
    "    #標題\n",
    "title_text = soup.select('.title.clearfix')[0].text.encode('utf-8')\n",
    "if \"《東森新聞雲》\" in title_text:\n",
    "    title = re.match(\"《東森新聞雲》(.*)\", title_text).group(1)\n",
    "else:\n",
    "    title = title_text\n",
    "    \n",
    "    #內容\n",
    "content = soup.select('.story > sectione > p')[1:]\n",
    "len(content)\n",
    "for i in xrange(0,len(content)):\n",
    "    for con in content[i].select('img'):\n",
    "        content[i].img.decompose()\n",
    "\n",
    "    #記者\n",
    "jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "\n",
    "\n",
    "#扣除圖片敘述  -->沒辦法\n",
    "#fb讚數  -->是粉絲團的\n",
    "#fb commenter\n",
    "\n",
    "#fb comment\n",
    "#fb comment time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 完整版-直接輸入從今天算起要抓幾天 ETToday(days,sleep_time=0)\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import math\n",
    "import re\n",
    "from datetime import date,datetime, timedelta \n",
    "import time\n",
    "\n",
    "def connect(dbname,password,host,port=3306,user):\n",
    "    import MySQLdb\n",
    "    # 承德電腦 host=\"10.120.30.1\"\n",
    "    db = MySQLdb.connect(host=host,user=\"root\",port=port,passwd=password,db=dbname,charset=\"utf8\")\n",
    "\n",
    "\n",
    "def insert_news(category,title,newsDate,journalist,content,hit=None,url,source,correct=1,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into news('class','title','newsDate','journalist','content','hit','url','source') values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),[category,title,newsDate,journalist,content,hit,url,source])\n",
    "    db.commit()\n",
    "\n",
    "def insert_comments(url,commenter,comment,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into comments('url','commenter','comment') values(%s,%s,%s),[url,commenter,comment])\n",
    "    db.commit()\n",
    "    \n",
    "def close:\n",
    "    db.close()\n",
    "\n",
    "def ETToday(days,sleep_time=0):\n",
    "    #day = str(date.today()- timedelta(1))\n",
    "    \n",
    "    ds = []\n",
    "    for i in xrange(1,days+1):\n",
    "        ds.append(str(date.today()- timedelta(i)))\n",
    "    \n",
    "    for day in ds:\n",
    "        urls_dic = {\"政治\":\"http://www.ettoday.net/news/news-list-%s-1.htm\"%day,\n",
    "                    \"財經\":\"http://www.ettoday.net/news/news-list-%s-17.htm\"%day,\n",
    "                    \"國際\":\"http://www.ettoday.net/news/news-list-%s-2.htm\"%day,\n",
    "                    \"大陸\":\"http://www.ettoday.net/news/news-list-%s-3.htm\"%day,\n",
    "                    \"社會\":\"http://www.ettoday.net/news/news-list-%s-6.htm\"%day,\n",
    "                    \"地方\":\"http://www.ettoday.net/news/news-list-%s-7.htm\"%day,\n",
    "                    \"影劇\":\"http://www.ettoday.net/news/news-list-%s-9.htm\"%day,\n",
    "                    \"3C\":\"http://www.ettoday.net/news/news-list-%s-20.htm\"%day\n",
    "                   }\n",
    "\n",
    "        L = []\n",
    "        for url in urls_dic:\n",
    "            res = requests.get(urls_dic[url])\n",
    "            res.encoding=\"utf-8\"\n",
    "            soup = BeautifulSoup(res.text)\n",
    "            page = re.search('\\((\\d+).\\)',soup.select(\".info\")[0].text).group(1)\n",
    "            pageCount = int(math.ceil(float(page)/30))\n",
    "            for block in soup.select(\"#all-news-list > h3\"):\n",
    "                title=block.select('a')[0].text\n",
    "                aLink=block.select('a')[0]['href']\n",
    "                content = soup.select('.story > sectione > p')[1:]\n",
    "                for i in xrange(0,len(content)):\n",
    "                    for con in content[i].select('img'):\n",
    "                        content[i].img.decompose()\n",
    "                jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "                journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "                L.append({'Category':block.select('em')[0].text,'Title':title,'Content':content,'Journalist':journalist,'Link':aLink,'Time':day,'Source':'ETToday'})\n",
    "                connect(\"news\",\"\",\"localhost\",\"root\")\n",
    "                insert_news()\n",
    "                close()\n",
    "                connect(\"comments\",\"\",\"localhost\",\"root\")\n",
    "                insert_comments()\n",
    "                close()\n",
    "                time.sleep(sleep_time)\n",
    "            \n",
    "            if pageCount > 1:\n",
    "                for i in xrange(2,pageCount+1):\n",
    "                    url_1=str(urls_dic[url]).split(\".htm\")[0] + \"-\" + str(i) + \".htm\"\n",
    "                    res = requests.get(url_1)\n",
    "                    res.encoding=\"utf-8\"\n",
    "                    soup = BeautifulSoup(res.text)\n",
    "                    for block in soup.select(\"#all-news-list > h3\"):\n",
    "                        title=block.select('a')[0].text\n",
    "                        aLink=block.select('a')[0]['href']\n",
    "                        content = soup.select('.story > sectione > p')[1:]\n",
    "                        for i in xrange(0,len(content)):\n",
    "                            for con in content[i].select('img'):\n",
    "                                content[i].img.decompose()\n",
    "                        jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "                        journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "                        L.append({'Category':block.select('em')[0].text,'Title':title,'Content':content,'Journalist':journalist,'Link':aLink,'Time':day,'Source':'ETToday'})\n",
    "                        connect(\"news\",\"\",\"localhost\",\"root\")\n",
    "                        insert_news()\n",
    "                        close()\n",
    "                        connect(\"comments\",\"\",\"localhost\",\"root\")\n",
    "                        insert_comments()\n",
    "                        close()\n",
    "                        time.sleep(sleep_time)\n",
    "            \n",
    "    # pandas\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(L,columns =['Category','Title','Content','Journalist','Link','Time','Source'])\n",
    "\n",
    "    # write in csv\n",
    "    f=open(\"ETToday.csv\",\"w\")\n",
    "    df.to_csv(\"ETToday\",encoding=\"utf-8\")\n",
    "    f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-178-65c1b7bb2b8d>, line 22)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-178-65c1b7bb2b8d>\"\u001b[0;36m, line \u001b[0;32m22\u001b[0m\n\u001b[0;31m    cur.execute(insert into news values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),[category,title,newsDate,journalist,content,hit,url,source,correct,remark])\u001b[0m\n\u001b[0m                          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "# 完整版-抓取單一日期或日期區間  ETToday_day(startday,endday,sleep_time=0) \n",
    "# sleep_time意思為抓取每一天新聞之間要休息多久，單位為秒\n",
    "# startday, endday 要輸入string(ex. '2015-1-1')\n",
    "# 沒輸入日期預設抓取前一天新聞\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import math\n",
    "import re\n",
    "from datetime import date,datetime, timedelta \n",
    "import time\n",
    "\n",
    "\n",
    "def connect(dbname,password,host,port=3306,user):\n",
    "    import MySQLdb\n",
    "    # 承德電腦 host=\"10.120.30.1\"\n",
    "    db = MySQLdb.connect(host=host,user=\"root\",port=port,passwd=password,db=dbname,charset=\"utf8\")\n",
    "\n",
    "\n",
    "def insert_news(category,title,newsDate,journalist,content,hit=None,url,source,correct=1,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into news('class','title','newsDate','journalist','content','hit','url','source') values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),[category,title,newsDate,journalist,content,hit,url,source])\n",
    "    db.commit()\n",
    "\n",
    "def insert_comments(url,commenter,comment,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into comments('url','commenter','comment') values(%s,%s,%s),[url,commenter,comment])\n",
    "    db.commit()\n",
    "    \n",
    "    \n",
    "def close:\n",
    "    db.close()\n",
    "\n",
    "sentinel = object()\n",
    "def ETToday_(startday=(str(date.today()-timedelta(1))),endday=sentinel,sleep_time=0):\n",
    "     \n",
    "    if endday is sentinel:\n",
    "        endday=startday\n",
    "    \n",
    "    def convert_day(d):\n",
    "        return datetime.strptime(d, '%Y-%m-%d')\n",
    "    \n",
    "    ed=convert_day(endday)\n",
    "    sd=convert_day(startday)\n",
    "\n",
    "    assert ed >= sd, \"endday must after startday\"\n",
    "\n",
    "    ds = []\n",
    "    for i in xrange(0,(ed-sd).days+1):\n",
    "        ds.append(str((ed- timedelta(i)).strftime('%Y-%m-%d')))\n",
    "    \n",
    "    for day in ds:\n",
    "        urls_dic = {\"政治\":\"http://www.ettoday.net/news/news-list-%s-1.htm\"%day,\n",
    "                    \"財經\":\"http://www.ettoday.net/news/news-list-%s-17.htm\"%day,\n",
    "                    \"國際\":\"http://www.ettoday.net/news/news-list-%s-2.htm\"%day,\n",
    "                    \"大陸\":\"http://www.ettoday.net/news/news-list-%s-3.htm\"%day,\n",
    "                    \"社會\":\"http://www.ettoday.net/news/news-list-%s-6.htm\"%day,\n",
    "                    \"地方\":\"http://www.ettoday.net/news/news-list-%s-7.htm\"%day,\n",
    "                    \"影劇\":\"http://www.ettoday.net/news/news-list-%s-9.htm\"%day,\n",
    "                    \"3C\":\"http://www.ettoday.net/news/news-list-%s-20.htm\"%day\n",
    "                   }\n",
    "\n",
    "        L = []\n",
    "        for url in urls_dic:\n",
    "            res = requests.get(urls_dic[url])\n",
    "            res.encoding=\"utf-8\"\n",
    "            soup = BeautifulSoup(res.text)\n",
    "            page = re.search('\\((\\d+).\\)',soup.select(\".info\")[0].text).group(1)\n",
    "            pageCount = int(math.ceil(float(page)/30))\n",
    "            for block in soup.select(\"#all-news-list > h3\"):\n",
    "                title=block.select('a')[0].text\n",
    "                aLink=block.select('a')[0]['href']\n",
    "                content = soup.select('.story > sectione > p')[1:]\n",
    "                for i in xrange(0,len(content)):\n",
    "                    for con in content[i].select('img'):\n",
    "                        content[i].img.decompose()\n",
    "                jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "                journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "                L.append({'Category':block.select('em')[0].text,'Title':title,'Content':content,'Journalist':journalist,'Link':aLink,'Time':day,'Source':'ETToday'})\n",
    "                connect(\"news\",\"\",\"localhost\",\"root\")\n",
    "                insert_news()\n",
    "                close()\n",
    "                connect(\"comments\",\"\",\"localhost\",\"root\")\n",
    "                insert_comments()\n",
    "                close()\n",
    "                time.sleep(sleep_time)\n",
    "            \n",
    "            if pageCount > 1:\n",
    "                for i in xrange(2,pageCount+1):\n",
    "                    url_1=str(urls_dic[url]).split(\".htm\")[0] + \"-\" + str(i) + \".htm\"\n",
    "                    res = requests.get(url_1)\n",
    "                    res.encoding=\"utf-8\"\n",
    "                    soup = BeautifulSoup(res.text)\n",
    "                    for block in soup.select(\"#all-news-list > h3\"):\n",
    "                        title=block.select('a')[0].text\n",
    "                        aLink=block.select('a')[0]['href']\n",
    "                        content = soup.select('.story > sectione > p')[1:]\n",
    "                        for i in xrange(0,len(content)):\n",
    "                            for con in content[i].select('img'):\n",
    "                                content[i].img.decompose()\n",
    "                        jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "                        journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "                        L.append({'Category':block.select('em')[0].text,'Title':title,'Content':content,'Journalist':journalist,'Link':aLink,'Time':day,'Source':'ETToday'})\n",
    "                        connect(\"news\",\"\",\"localhost\",\"root\")\n",
    "                        insert_news()\n",
    "                        close()\n",
    "                        connect(\"comments\",\"\",\"localhost\",\"root\")\n",
    "                        insert_comments()\n",
    "                        close()\n",
    "                        time.sleep(sleep_time)\n",
    "            \n",
    "    # pandas\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(L,columns =['Category','Title','Content','Journalist','Link','Time','Source'])\n",
    "\n",
    "    # write in csv\n",
    "    if startday==endday:\n",
    "        std=\"\".join(startday.split('-'))\n",
    "        filename=\"ETToday\"+\"_\"+std+\".csv\"\n",
    "    else:\n",
    "        std=\"\".join(startday.split('-'))\n",
    "        end=\"\".join(endday.split('-'))\n",
    "        filename=\"ETToday\"+\"_\"+std+\"-\"+end+\".csv\"\n",
    "    print filename\n",
    "    f=open(filename,\"w\")\n",
    "    df.to_csv(filename,encoding=\"utf-8\")\n",
    "    f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "http://www.ettoday.net/news/news-list-2015-01-05-20.htm\n",
      "http://www.ettoday.net/news/news-list-2015-01-04-20.htm\n",
      "http://www.ettoday.net/news/news-list-2015-01-03-20.htm\n",
      "http://www.ettoday.net/news/news-list-2015-01-02-20.htm\n",
      "http://www.ettoday.net/news/news-list-2015-01-01-20.htm\n"
     ]
    }
   ],
   "source": [
    "# 完整版-抓取單一或多月份  ETToday_month(year,month,year2=year,month2=month,sleep_time=0) \n",
    "# sleep_time意思為抓取每一天新聞之間要休息多久，單位為秒\n",
    "# year, month要輸入integer(ex. 2012,3)\n",
    "# 如果抓本月的新聞，會預設抓到今天前一天的新聞\n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import math\n",
    "import re\n",
    "from datetime import date,datetime, timedelta \n",
    "import time\n",
    "import calendar\n",
    "\n",
    "def connect(dbname,password,host,port=3306,user):\n",
    "    import MySQLdb\n",
    "    # 承德電腦 host=\"10.120.30.1\"\n",
    "    db = MySQLdb.connect(host=host,user=\"root\",port=port,passwd=password,db=dbname,charset=\"utf8\")\n",
    "    \n",
    "def insert_news(category,title,newsDate,journalist,content,hit=None,url,source,correct=1,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into news('class','title','newsDate','journalist','content','hit','url','source') values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),[category,title,newsDate,journalist,content,hit,url,source])\n",
    "    db.commit()\n",
    "\n",
    "def insert_comments(url,commenter,comment,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into comments('url','commenter','comment') values(%s,%s,%s),[url,commenter,comment])\n",
    "    db.commit()\n",
    "    \n",
    "def close:\n",
    "    db.close()\n",
    "\n",
    "sentinel = object()\n",
    "def ETToday_month(year,month,year2=sentinel,month2=sentinel,sleep_time=0):\n",
    "    \n",
    "    if year2 is sentinel:\n",
    "        year2 = year\n",
    "    if month2 is sentinel:\n",
    "        month2 = month\n",
    "    \n",
    "    assert date(year2,month2,1)>=date(year,month,1), \"year2,month2 must after year,month\"\n",
    "\n",
    "    def first_day_of_month_d(d):\n",
    "        return date(d.year, d.month, 1)\n",
    "\n",
    "    def first_day_of_month(y,m):\n",
    "        return date(y,m,1)\n",
    "\n",
    "    def last_day_of_month(y,m):\n",
    "        return date(y,m+1,1) - timedelta(days = 1)\n",
    "\n",
    "    def diff_months_d(d1, d2):\n",
    "        return (d1.year - d2.year)*12 + d1.month - d2.month\n",
    "\n",
    "    def diff_days(y,m,y2,m2):\n",
    "        return abs(last_day_of_month(y2,m2)-first_day_of_month(y,m)).days\n",
    "\n",
    "    # check if input month exceed current month\n",
    "    assert diff_months_d(date.today(),first_day_of_month(year,month))>=0, \"眼睛睜大點好嘛？%s月還沒到ok？\"%month\n",
    "    assert diff_months_d(date.today(),first_day_of_month(year2,month2))>=0, \"眼睛睜大點好嘛？%s月還沒到ok？\"%month2\n",
    "\n",
    "    if diff_months_d(date.today(),first_day_of_month(year2,month2))==0:\n",
    "        duration=(date.today()-first_day_of_month_d(date.today())).days\n",
    "        ds = []\n",
    "        for i in xrange(1,duration+1):\n",
    "            ds.append(str((date.today()- timedelta(i)).strftime('%Y-%m-%d')))\n",
    "    else:\n",
    "        #duration=calendar.monthrange(year,month)[1]\n",
    "        duration=diff_days(year,month,year2,month2)\n",
    "        ds = []\n",
    "        for i in xrange(0,duration+1):\n",
    "            ds.append(str((last_day_of_month(year2,month2)- timedelta(i)).strftime('%Y-%m-%d')))\n",
    "\n",
    "    for day in ds:\n",
    "        urls_dic = {\"政治\":\"http://www.ettoday.net/news/news-list-%s-1.htm\"%day,\n",
    "                    \"財經\":\"http://www.ettoday.net/news/news-list-%s-17.htm\"%day,\n",
    "                    \"國際\":\"http://www.ettoday.net/news/news-list-%s-2.htm\"%day,\n",
    "                    \"大陸\":\"http://www.ettoday.net/news/news-list-%s-3.htm\"%day,\n",
    "                    \"社會\":\"http://www.ettoday.net/news/news-list-%s-6.htm\"%day,\n",
    "                    \"地方\":\"http://www.ettoday.net/news/news-list-%s-7.htm\"%day,\n",
    "                    \"影劇\":\"http://www.ettoday.net/news/news-list-%s-9.htm\"%day,\n",
    "                    \"3C\":\"http://www.ettoday.net/news/news-list-%s-20.htm\"%day\n",
    "                   }\n",
    "\n",
    "        L = []\n",
    "        for url in urls_dic:\n",
    "            res = requests.get(urls_dic[url])\n",
    "            res.encoding=\"utf-8\"\n",
    "            soup = BeautifulSoup(res.text)\n",
    "            page = re.search('\\((\\d+).\\)',soup.select(\".info\")[0].text).group(1)\n",
    "            pageCount = int(math.ceil(float(page)/30))\n",
    "            for block in soup.select(\"#all-news-list > h3\"):\n",
    "                title=block.select('a')[0].text\n",
    "                aLink=block.select('a')[0]['href']\n",
    "                content = soup.select('.story > sectione > p')[1:]\n",
    "                for i in xrange(0,len(content)):\n",
    "                    for con in content[i].select('img'):\n",
    "                        content[i].img.decompose()\n",
    "                jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "                journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "                L.append({'Category':block.select('em')[0].text,'Title':title,'Content':content,'Journalist':journalist,'Link':aLink,'Time':day,'Source':'ETToday'})\n",
    "                connect(\"news\",\"\",\"localhost\",\"root\")\n",
    "                insert_news()\n",
    "                close()\n",
    "                connect(\"comments\",\"\",\"localhost\",\"root\")\n",
    "                insert_comments()\n",
    "                close()\n",
    "                time.sleep(sleep_time)\n",
    "            \n",
    "            if pageCount > 1:\n",
    "                for i in xrange(2,pageCount+1):\n",
    "                    url_1=str(urls_dic[url]).split(\".htm\")[0] + \"-\" + str(i) + \".htm\"\n",
    "                    res = requests.get(url_1)\n",
    "                    res.encoding=\"utf-8\"\n",
    "                    soup = BeautifulSoup(res.text)\n",
    "                    for block in soup.select(\"#all-news-list > h3\"):\n",
    "                        title=block.select('a')[0].text\n",
    "                        aLink=block.select('a')[0]['href']\n",
    "                        content = soup.select('.story > sectione > p')[1:]\n",
    "                        for i in xrange(0,len(content)):\n",
    "                            for con in content[i].select('img'):\n",
    "                                content[i].img.decompose()\n",
    "                        jour = str(soup.select('.story > sectione > p')[0].text.encode('utf-8'))\n",
    "                        journalist = (jour.split(\"記者\")[1]).split(\"／\")[0]\n",
    "                        L.append({'Category':block.select('em')[0].text,'Title':title,'Content':content,'Journalist':journalist,'Link':aLink,'Time':day,'Source':'ETToday'})\n",
    "                        connect(\"news\",\"\",\"localhost\",\"root\")\n",
    "                        insert_news()\n",
    "                        close()\n",
    "                        connect(\"comments\",\"\",\"localhost\",\"root\")\n",
    "                        insert_comments()\n",
    "                        close()\n",
    "                        time.sleep(sleep_time)\n",
    "            \n",
    "    # pandas\n",
    "    import pandas as pd\n",
    "    df = pd.DataFrame(L,columns =['Category','Title','Content','Journalist','Link','Time','Source'])\n",
    "\n",
    "    # write in csv\n",
    "    if (str(year)+str(month))==(str(year2)+str(month2)):\n",
    "        filename=\"ETTdoay_%d%02d.csv\"%(year,month)\n",
    "    else:\n",
    "        filename=\"ETTdoay_%d%02d-%d%02d.csv\"%(year,month,year2,month2)\n",
    "    f=open(filename,\"w\")\n",
    "    df.to_csv(filename,encoding=\"utf-8\")\n",
    "    f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# run\n",
    "ETToday_links(1)\n",
    "ETToday_links_day('2015-06-14')\n",
    "ETToday_links_C(1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Category</th>\n",
       "      <th>Title</th>\n",
       "      <th>Link</th>\n",
       "      <th>Time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>社會</td>\n",
       "      <td>快訊／七堵行政大樓停車場　男在車內燒炭亡</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520775.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>社會</td>\n",
       "      <td>勇消忍800度高溫無力跪地　吸幾口氧氣後...再回頭滅火</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520711.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>社會</td>\n",
       "      <td>快訊／不滿女友提分手　越男攀爬桃市府跳樓</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520708.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>社會</td>\n",
       "      <td>也是災情？情侶騎車遇暴雨　男自己穿雨衣女暴怒打他</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520737.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>社會</td>\n",
       "      <td>擔心遭撿屍…竹科女喝醉到超商休息　色男猥褻還洗劫</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520700.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>社會</td>\n",
       "      <td>替代役男割草猝死　母自責：教兒忍耐，笨媽反害死他</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520724.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>社會</td>\n",
       "      <td>快訊／騎單車被轉彎砂石車輾過　屏東鹽埔母女雙亡</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520690.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>社會</td>\n",
       "      <td>為平板推同學撞車？少年母：玩笑話，不小心拉扯</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520640.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>社會</td>\n",
       "      <td>美好回憶毀了...惡鄰占地拆房變賣　阿嬤老家變停車格</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520621.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>社會</td>\n",
       "      <td>2屁孩拒攔檢快閃鑽小巷　警連開28張紅單罰3萬</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520593.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>社會</td>\n",
       "      <td>國中兒遭推撞車臉半毀不敢喊痛　母：心快痛死了</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520596.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>社會</td>\n",
       "      <td>戴「豬頭」面具偷遍桃園　警看監視器畫面忍不住笑了</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520554.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>社會</td>\n",
       "      <td>「你死了平板就我的」　國二生推同學撞車挨告</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520531.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>社會</td>\n",
       "      <td>牙齒磨成粉末驗DNA　鑑識專家：堅固不易毀壞骨髓易保存</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520516.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>社會</td>\n",
       "      <td>想吃狗肉強行拖走家犬　越南移工強辯「沒吃到」送辦</td>\n",
       "      <td>http://www.ettoday.net/news/20150614/520440.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>社會</td>\n",
       "      <td>見檳榔西施「胸器」逼人　色狼凍未條強「摸兩粒」</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520400.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>社會</td>\n",
       "      <td>追來了！載兒超併排車遭亮槍「砰砰」射車　駕駛嚇到麻</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520488.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>社會</td>\n",
       "      <td>五子命案白骨DNA報告確認後結案　若非劉姓夫妻將併查</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520434.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>社會</td>\n",
       "      <td>快訊／五子命案白骨拼圖　傳DNA結果是劉志勤、林真米</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520079.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>社會</td>\n",
       "      <td>這你也信？男誆「老蔣5千噸黃金存單」　騙到210萬</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520369.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>社會</td>\n",
       "      <td>濃濃黑煙！小琉球瓦斯氣爆　3員工灼傷送醫</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520345.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>社會</td>\n",
       "      <td>詐領司機薪水給女兒繳貸款　雲林口湖鄉長遭羈押禁見</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520339.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>社會</td>\n",
       "      <td>半年做一次...夫迷上養生館E奶按摩女　人妻被冷落報警</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520304.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>社會</td>\n",
       "      <td>成功嶺役男烈日下割草暴斃　母：兒曾反應「在地獄」</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520248.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>社會</td>\n",
       "      <td>19歲男1年騙4千萬　抓大把鈔票PO臉書炫富</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520294.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>社會</td>\n",
       "      <td>62歲婦搶78歲翁7200元生活費　「點點衣」露餡被逮</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520288.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>社會</td>\n",
       "      <td>快訊／板橋工地驚傳鷹架倒塌　1人受傷送醫1人受困中</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520293.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>社會</td>\n",
       "      <td>癡戀男跟蹤騷擾5年　黃女拿到保護令：這真的有效嗎？</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520241.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>社會</td>\n",
       "      <td>五子命案最後一塊拼圖？兩具白骨如果不是劉志勤夫婦…</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520238.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>社會</td>\n",
       "      <td>6月大女嬰放冰櫃「涼一下」引罵　老爸澄清只放幾秒</td>\n",
       "      <td>http://www.ettoday.net/news/20150613/520229.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1426</th>\n",
       "      <td>影劇</td>\n",
       "      <td>女兒問「昨天第一次做愛」該怎麼回？　許常德這樣答</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517883.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1427</th>\n",
       "      <td>影劇</td>\n",
       "      <td>昆凌5個月孕肚藏不住　襯衫釦子卡在「小周周」上</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517852.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1428</th>\n",
       "      <td>影劇</td>\n",
       "      <td>小安出國去天堂了...　安以軒淚崩約定：我們下輩子見</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517880.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1429</th>\n",
       "      <td>影劇</td>\n",
       "      <td>大小S淚崩送別安鈞璨　滿眼淚水哭到說不出話</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517882.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1430</th>\n",
       "      <td>影劇</td>\n",
       "      <td>竇靖童年底日本出道　傳將在王菲巡演舞台上獻唱</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517879.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1431</th>\n",
       "      <td>影劇</td>\n",
       "      <td>凌晨4點玻璃門自爆　胡宇威嚇傻：我是撞鬼嗎？</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517867.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1432</th>\n",
       "      <td>影劇</td>\n",
       "      <td>金陽洗澡ing...郭書瑤坐等門外　大吃愛狗飛醋</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517851.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1433</th>\n",
       "      <td>影劇</td>\n",
       "      <td>對歐陽3姊妹因材施教！傅娟：找出最適合孩子的方式</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517848.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1434</th>\n",
       "      <td>影劇</td>\n",
       "      <td>「影帝」謝佳見歸零闖台灣　暖男發電機惹人心跳漏拍</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/509676.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1435</th>\n",
       "      <td>影劇</td>\n",
       "      <td>直擊／安鈞璨告別式　白歆惠不捨淚：來送小安回家！</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517824.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1436</th>\n",
       "      <td>影劇</td>\n",
       "      <td>快訊／安鈞璨告別式　姊姊安以軒遠赴台南送他最後一程</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517814.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1437</th>\n",
       "      <td>影劇</td>\n",
       "      <td>有關注有保庇！陳柏霖留言命中大陸高考作文題</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517779.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1438</th>\n",
       "      <td>影劇</td>\n",
       "      <td>他們憑什麼搶占魅力才神？「這些歌」肯定讓你佩服不已</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/509083.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1439</th>\n",
       "      <td>影劇</td>\n",
       "      <td>師大附中畢業名人錄賀詞　馬西屏驚：你們是名嘴搖籃！</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517782.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1440</th>\n",
       "      <td>影劇</td>\n",
       "      <td>被控暴力性侵　趙駿亞感嘆：她是同居2個月的女朋友</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517774.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1441</th>\n",
       "      <td>影劇</td>\n",
       "      <td>趙駿亞被控暴力性侵　高宇蓁驚訝：是不是搞錯人！</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517762.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1442</th>\n",
       "      <td>影劇</td>\n",
       "      <td>布裘出國度假攜6子搭經濟艙　超低調自搬行李超親民</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517752.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1443</th>\n",
       "      <td>影劇</td>\n",
       "      <td>「八爺」鄭嘉穎太會演　多變戲路奪「經典男神」亞軍</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/508311.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1444</th>\n",
       "      <td>影劇</td>\n",
       "      <td>趙駿亞酒醉後暴力性侵？　遭女友人控撕衣、強迫口交</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517742.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1445</th>\n",
       "      <td>影劇</td>\n",
       "      <td>小產3次不放棄　美女主播陳海茵結婚12年終又懷孕！</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517722.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1446</th>\n",
       "      <td>影劇</td>\n",
       "      <td>袁艾菲&amp;周定緯又私約出遊　浪漫穿情侶裝、看電影</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517716.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1447</th>\n",
       "      <td>影劇</td>\n",
       "      <td>17歲新人中斷學業闖歌壇！王治平：想讀書永遠不嫌晚</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517693.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1448</th>\n",
       "      <td>影劇</td>\n",
       "      <td>斷開女女戀！劉嘉玲梁朝偉合體曬恩愛　分居風暴解除</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517695.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1449</th>\n",
       "      <td>影劇</td>\n",
       "      <td>特稿／安鈞璨與助理不只是一張合約　離別前比家人還親</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517628.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1450</th>\n",
       "      <td>影劇</td>\n",
       "      <td>八爺鄭嘉穎傳求婚成功！　KTV捧心型戒指套牢23歲女友</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517689.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1451</th>\n",
       "      <td>影劇</td>\n",
       "      <td>棒棒堂小祿Battle尬舞　屏東大學校花穿短裙出招秒獲勝</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517680.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1452</th>\n",
       "      <td>影劇</td>\n",
       "      <td>李奧納多遭爆搞大蕾哈娜的肚子！八卦雜誌亂說話完蛋了</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517690.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1453</th>\n",
       "      <td>影劇</td>\n",
       "      <td>《民歌40》台北場超時演唱　接連2天被罰近80萬元</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517686.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454</th>\n",
       "      <td>影劇</td>\n",
       "      <td>林美秀回到40年前　綁雙辮撞臉小丸子</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517659.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455</th>\n",
       "      <td>影劇</td>\n",
       "      <td>世界第二小國每人每月賺48萬　浩翔被滿街超跑嚇到驚呆</td>\n",
       "      <td>http://www.ettoday.net/news/20150608/517657.htm</td>\n",
       "      <td>2015-06-14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1456 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Category                          Title  \\\n",
       "0          社會          快訊／七堵行政大樓停車場　男在車內燒炭亡    \n",
       "1          社會  勇消忍800度高溫無力跪地　吸幾口氧氣後...再回頭滅火    \n",
       "2          社會          快訊／不滿女友提分手　越男攀爬桃市府跳樓    \n",
       "3          社會      也是災情？情侶騎車遇暴雨　男自己穿雨衣女暴怒打他    \n",
       "4          社會      擔心遭撿屍…竹科女喝醉到超商休息　色男猥褻還洗劫    \n",
       "5          社會      替代役男割草猝死　母自責：教兒忍耐，笨媽反害死他    \n",
       "6          社會       快訊／騎單車被轉彎砂石車輾過　屏東鹽埔母女雙亡    \n",
       "7          社會        為平板推同學撞車？少年母：玩笑話，不小心拉扯    \n",
       "8          社會    美好回憶毀了...惡鄰占地拆房變賣　阿嬤老家變停車格    \n",
       "9          社會       2屁孩拒攔檢快閃鑽小巷　警連開28張紅單罰3萬    \n",
       "10         社會        國中兒遭推撞車臉半毀不敢喊痛　母：心快痛死了    \n",
       "11         社會      戴「豬頭」面具偷遍桃園　警看監視器畫面忍不住笑了    \n",
       "12         社會         「你死了平板就我的」　國二生推同學撞車挨告    \n",
       "13         社會   牙齒磨成粉末驗DNA　鑑識專家：堅固不易毀壞骨髓易保存    \n",
       "14         社會      想吃狗肉強行拖走家犬　越南移工強辯「沒吃到」送辦    \n",
       "15         社會       見檳榔西施「胸器」逼人　色狼凍未條強「摸兩粒」    \n",
       "16         社會     追來了！載兒超併排車遭亮槍「砰砰」射車　駕駛嚇到麻    \n",
       "17         社會    五子命案白骨DNA報告確認後結案　若非劉姓夫妻將併查    \n",
       "18         社會    快訊／五子命案白骨拼圖　傳DNA結果是劉志勤、林真米    \n",
       "19         社會     這你也信？男誆「老蔣5千噸黃金存單」　騙到210萬    \n",
       "20         社會          濃濃黑煙！小琉球瓦斯氣爆　3員工灼傷送醫    \n",
       "21         社會      詐領司機薪水給女兒繳貸款　雲林口湖鄉長遭羈押禁見    \n",
       "22         社會   半年做一次...夫迷上養生館E奶按摩女　人妻被冷落報警    \n",
       "23         社會      成功嶺役男烈日下割草暴斃　母：兒曾反應「在地獄」    \n",
       "24         社會        19歲男1年騙4千萬　抓大把鈔票PO臉書炫富    \n",
       "25         社會   62歲婦搶78歲翁7200元生活費　「點點衣」露餡被逮    \n",
       "26         社會     快訊／板橋工地驚傳鷹架倒塌　1人受傷送醫1人受困中    \n",
       "27         社會     癡戀男跟蹤騷擾5年　黃女拿到保護令：這真的有效嗎？    \n",
       "28         社會     五子命案最後一塊拼圖？兩具白骨如果不是劉志勤夫婦…    \n",
       "29         社會      6月大女嬰放冰櫃「涼一下」引罵　老爸澄清只放幾秒    \n",
       "...       ...                            ...   \n",
       "1426       影劇      女兒問「昨天第一次做愛」該怎麼回？　許常德這樣答    \n",
       "1427       影劇       昆凌5個月孕肚藏不住　襯衫釦子卡在「小周周」上    \n",
       "1428       影劇    小安出國去天堂了...　安以軒淚崩約定：我們下輩子見    \n",
       "1429       影劇         大小S淚崩送別安鈞璨　滿眼淚水哭到說不出話    \n",
       "1430       影劇        竇靖童年底日本出道　傳將在王菲巡演舞台上獻唱    \n",
       "1431       影劇        凌晨4點玻璃門自爆　胡宇威嚇傻：我是撞鬼嗎？    \n",
       "1432       影劇      金陽洗澡ing...郭書瑤坐等門外　大吃愛狗飛醋    \n",
       "1433       影劇      對歐陽3姊妹因材施教！傅娟：找出最適合孩子的方式    \n",
       "1434       影劇      「影帝」謝佳見歸零闖台灣　暖男發電機惹人心跳漏拍    \n",
       "1435       影劇      直擊／安鈞璨告別式　白歆惠不捨淚：來送小安回家！    \n",
       "1436       影劇     快訊／安鈞璨告別式　姊姊安以軒遠赴台南送他最後一程    \n",
       "1437       影劇         有關注有保庇！陳柏霖留言命中大陸高考作文題    \n",
       "1438       影劇     他們憑什麼搶占魅力才神？「這些歌」肯定讓你佩服不已    \n",
       "1439       影劇     師大附中畢業名人錄賀詞　馬西屏驚：你們是名嘴搖籃！    \n",
       "1440       影劇      被控暴力性侵　趙駿亞感嘆：她是同居2個月的女朋友    \n",
       "1441       影劇       趙駿亞被控暴力性侵　高宇蓁驚訝：是不是搞錯人！    \n",
       "1442       影劇      布裘出國度假攜6子搭經濟艙　超低調自搬行李超親民    \n",
       "1443       影劇      「八爺」鄭嘉穎太會演　多變戲路奪「經典男神」亞軍    \n",
       "1444       影劇      趙駿亞酒醉後暴力性侵？　遭女友人控撕衣、強迫口交    \n",
       "1445       影劇     小產3次不放棄　美女主播陳海茵結婚12年終又懷孕！    \n",
       "1446       影劇       袁艾菲&周定緯又私約出遊　浪漫穿情侶裝、看電影    \n",
       "1447       影劇     17歲新人中斷學業闖歌壇！王治平：想讀書永遠不嫌晚    \n",
       "1448       影劇      斷開女女戀！劉嘉玲梁朝偉合體曬恩愛　分居風暴解除    \n",
       "1449       影劇     特稿／安鈞璨與助理不只是一張合約　離別前比家人還親    \n",
       "1450       影劇   八爺鄭嘉穎傳求婚成功！　KTV捧心型戒指套牢23歲女友    \n",
       "1451       影劇  棒棒堂小祿Battle尬舞　屏東大學校花穿短裙出招秒獲勝    \n",
       "1452       影劇     李奧納多遭爆搞大蕾哈娜的肚子！八卦雜誌亂說話完蛋了    \n",
       "1453       影劇     《民歌40》台北場超時演唱　接連2天被罰近80萬元    \n",
       "1454       影劇            林美秀回到40年前　綁雙辮撞臉小丸子    \n",
       "1455       影劇    世界第二小國每人每月賺48萬　浩翔被滿街超跑嚇到驚呆    \n",
       "\n",
       "                                                 Link        Time  \n",
       "0     http://www.ettoday.net/news/20150614/520775.htm  2015-06-14  \n",
       "1     http://www.ettoday.net/news/20150614/520711.htm  2015-06-14  \n",
       "2     http://www.ettoday.net/news/20150614/520708.htm  2015-06-14  \n",
       "3     http://www.ettoday.net/news/20150614/520737.htm  2015-06-14  \n",
       "4     http://www.ettoday.net/news/20150614/520700.htm  2015-06-14  \n",
       "5     http://www.ettoday.net/news/20150614/520724.htm  2015-06-14  \n",
       "6     http://www.ettoday.net/news/20150614/520690.htm  2015-06-14  \n",
       "7     http://www.ettoday.net/news/20150614/520640.htm  2015-06-14  \n",
       "8     http://www.ettoday.net/news/20150614/520621.htm  2015-06-14  \n",
       "9     http://www.ettoday.net/news/20150614/520593.htm  2015-06-14  \n",
       "10    http://www.ettoday.net/news/20150614/520596.htm  2015-06-14  \n",
       "11    http://www.ettoday.net/news/20150614/520554.htm  2015-06-14  \n",
       "12    http://www.ettoday.net/news/20150614/520531.htm  2015-06-14  \n",
       "13    http://www.ettoday.net/news/20150614/520516.htm  2015-06-14  \n",
       "14    http://www.ettoday.net/news/20150614/520440.htm  2015-06-14  \n",
       "15    http://www.ettoday.net/news/20150613/520400.htm  2015-06-14  \n",
       "16    http://www.ettoday.net/news/20150613/520488.htm  2015-06-14  \n",
       "17    http://www.ettoday.net/news/20150613/520434.htm  2015-06-14  \n",
       "18    http://www.ettoday.net/news/20150613/520079.htm  2015-06-14  \n",
       "19    http://www.ettoday.net/news/20150613/520369.htm  2015-06-14  \n",
       "20    http://www.ettoday.net/news/20150613/520345.htm  2015-06-14  \n",
       "21    http://www.ettoday.net/news/20150613/520339.htm  2015-06-14  \n",
       "22    http://www.ettoday.net/news/20150613/520304.htm  2015-06-14  \n",
       "23    http://www.ettoday.net/news/20150613/520248.htm  2015-06-14  \n",
       "24    http://www.ettoday.net/news/20150613/520294.htm  2015-06-14  \n",
       "25    http://www.ettoday.net/news/20150613/520288.htm  2015-06-14  \n",
       "26    http://www.ettoday.net/news/20150613/520293.htm  2015-06-14  \n",
       "27    http://www.ettoday.net/news/20150613/520241.htm  2015-06-14  \n",
       "28    http://www.ettoday.net/news/20150613/520238.htm  2015-06-14  \n",
       "29    http://www.ettoday.net/news/20150613/520229.htm  2015-06-14  \n",
       "...                                               ...         ...  \n",
       "1426  http://www.ettoday.net/news/20150608/517883.htm  2015-06-14  \n",
       "1427  http://www.ettoday.net/news/20150608/517852.htm  2015-06-14  \n",
       "1428  http://www.ettoday.net/news/20150608/517880.htm  2015-06-14  \n",
       "1429  http://www.ettoday.net/news/20150608/517882.htm  2015-06-14  \n",
       "1430  http://www.ettoday.net/news/20150608/517879.htm  2015-06-14  \n",
       "1431  http://www.ettoday.net/news/20150608/517867.htm  2015-06-14  \n",
       "1432  http://www.ettoday.net/news/20150608/517851.htm  2015-06-14  \n",
       "1433  http://www.ettoday.net/news/20150608/517848.htm  2015-06-14  \n",
       "1434  http://www.ettoday.net/news/20150608/509676.htm  2015-06-14  \n",
       "1435  http://www.ettoday.net/news/20150608/517824.htm  2015-06-14  \n",
       "1436  http://www.ettoday.net/news/20150608/517814.htm  2015-06-14  \n",
       "1437  http://www.ettoday.net/news/20150608/517779.htm  2015-06-14  \n",
       "1438  http://www.ettoday.net/news/20150608/509083.htm  2015-06-14  \n",
       "1439  http://www.ettoday.net/news/20150608/517782.htm  2015-06-14  \n",
       "1440  http://www.ettoday.net/news/20150608/517774.htm  2015-06-14  \n",
       "1441  http://www.ettoday.net/news/20150608/517762.htm  2015-06-14  \n",
       "1442  http://www.ettoday.net/news/20150608/517752.htm  2015-06-14  \n",
       "1443  http://www.ettoday.net/news/20150608/508311.htm  2015-06-14  \n",
       "1444  http://www.ettoday.net/news/20150608/517742.htm  2015-06-14  \n",
       "1445  http://www.ettoday.net/news/20150608/517722.htm  2015-06-14  \n",
       "1446  http://www.ettoday.net/news/20150608/517716.htm  2015-06-14  \n",
       "1447  http://www.ettoday.net/news/20150608/517693.htm  2015-06-14  \n",
       "1448  http://www.ettoday.net/news/20150608/517695.htm  2015-06-14  \n",
       "1449  http://www.ettoday.net/news/20150608/517628.htm  2015-06-14  \n",
       "1450  http://www.ettoday.net/news/20150608/517689.htm  2015-06-14  \n",
       "1451  http://www.ettoday.net/news/20150608/517680.htm  2015-06-14  \n",
       "1452  http://www.ettoday.net/news/20150608/517690.htm  2015-06-14  \n",
       "1453  http://www.ettoday.net/news/20150608/517686.htm  2015-06-14  \n",
       "1454  http://www.ettoday.net/news/20150608/517659.htm  2015-06-14  \n",
       "1455  http://www.ettoday.net/news/20150608/517657.htm  2015-06-14  \n",
       "\n",
       "[1456 rows x 4 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# insert sql\n",
    "def connect(dbname,password,host,port=3306,user,):\n",
    "    import MySQLdb\n",
    "    # 承德電腦 host=\"10.120.30.1\"\n",
    "    db = MySQLdb.connect(host=host,user=\"root\",port=port,passwd=password,db=dbname,charset=\"utf8\")\n",
    "\n",
    "def insert_news(category,title,newsDate,journalist,content,hit=None,url,source,correct=1,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into news values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s),[category,title,newsDate,journalist,content,hit,url,source,correct,remark])\n",
    "    db.commit()\n",
    "\n",
    "def insert_comments(url,commenter,comment,remark=None):\n",
    "    cur = db.cursor()\n",
    "    cur.execute(insert into comments values(%s,%s,%s,%s),[url,commenter,comment,remark])\n",
    "    db.commit()\n",
    "    \n",
    "def close:\n",
    "    db.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopword=[\"ETtoday\",\"東森新聞雲\",\"直擊／\",\"快訊／\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 蘋果日報"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<script src=\"http://twimg.edgesuite.net/www/js/html5.js\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/jquery-1.7.2.min.js\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/jquery.tools.dateinput.js\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/jquery.lightbox-0.5.min-imgpv.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/appledaily-detectmobilebrowser.js\" type=\"text/javascript\"></script>, <script type=\"text/javascript\">\n",
      "            function chartbeatvideo() {\n",
      "                var mobile = (/iphone|ipad|ipod|android|blackberry|xbox|mini|windows\\sce|palm/i.test(navigator.userAgent.toLowerCase()));\n",
      "                if (!mobile) {\n",
      "                   StrategyInterface.prototype.getTitle = function() {return '奇蹟 少女漂海21小時獲救';}\n",
      "                   StrategyInterface.prototype.getVideoPath = function() {return 'http://twimg.edgesuite.net/video/20150610/9fd9b6d2eb0f77ac7616.mp4';}\n",
      "                   window['_cbv_strategies'] = window['_cbv_strategies'] || [];\n",
      "                    window['_cbv_strategies'].push(StrategyInterface);\n",
      "                    var _cbv = window._cbv || (window._cbv = []);\n",
      "                     _cbv.push(document.getElementById(\"flow_player\").innerHTML);\n",
      "                }\n",
      "            }\n",
      "        </script>, <script language=\"JavaScript\" src=\"http://twimg.edgesuite.net/appledaily/images/js/OneAD_Article_Video_v1.js\" type=\"text/javascript\"></script>, <script language=\"JavaScript\" type=\"text/javascript\">\n",
      "        // For OneAD, DON'T MODIFY the following\n",
      "            if (ONEAD){\n",
      "                ONEAD.uid = \"1000001\";\n",
      "                ONEAD.external_url = \"http://advideoadmin.appledaily.com.tw/\"; // base_url, post-slash is necessary\n",
      "                ONEAD.wrapper = 'ONEAD_player_wrapper';\n",
      "                ONEAD.category = '3-appledaily_headline';\n",
      "                ONEAD.display_type = 'inpage'; // inpage | instream\n",
      "                //ONEAD.wmode = 'opaque';\n",
      "            }\n",
      "            if (typeof window.isip_js == \"undefined\") {\n",
      "                    // async\n",
      "                    (function() {\n",
      "                    var src = 'http://ad-specs.guoshipartners.com/static/js/isip.v2.js';\n",
      "                    var js = document.createElement('script');\n",
      "                    js.async = true;\n",
      "                    js.type = 'text/javascript';\n",
      "                    var useSSL = 'https:' == document.location.protocol;\n",
      "                    js.src = src;\n",
      "                    var node = document.getElementsByTagName('script')[0];\n",
      "                    node.parentNode.insertBefore(js, node.nextSibling); // insert after\n",
      "                    })();\n",
      "            }\n",
      "        </script>, <script language=\"JavaScript\" src=\"http://twimg.edgesuite.net/adx/gpt.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/adx1/appledailycomtw.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/adx1/flowplayer-3.2.10.min.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/adx1/flowplayer.ipad-3.2.8.min.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/adx1/flowplayer.con.js\" type=\"text/javascript\"></script>, <script>\n",
      "        googletag.cmd.push(function() {\n",
      "            HeadBanner = gpt.createAdSlot('AppleDaily/headline_detail/HeadBanner', 'HeadBanner', 'leaderboard',{'apple_daily_article_id':36599626});\n",
      "            LREC1 = gpt.createAdSlot('AppleDaily/headline_detail/LREC1', 'LREC', 'rectangleAD1',{'apple_daily_article_id':36599626});\n",
      "            LREC2 = gpt.createAdSlot('AppleDaily/headline_detail/LREC2', 'LREC', 'rectangleAD2',{'apple_daily_article_id':36599626});\n",
      "            W1 = gpt.createAdSlot('AppleDaily/headline_detail/Wallpaper1', '150x800', 'door-left');\n",
      "            W2 = gpt.createAdSlot('AppleDaily/headline_detail/Wallpaper2', '150x800', 'door-right');\n",
      "            articlelast = gpt.createAdSlot('AppleDaily/headline_detail/articlelast', '300x250', 'articlelast',{'apple_daily_article_id':36599626});\n",
      "            articlelast2 = gpt.createAdSlot('AppleDaily/headline_detail/articlelast2', '300x250', 'articlelast2',{'apple_daily_article_id':36599626});\n",
      "            articlelast3 = gpt.createAdSlot('AppleDaily/headline_detail/articlelast3', '580x400', 'articlelast3',{'apple_daily_article_id':36599626});\n",
      "            textlink = gpt.createAdSlot('AppleDaily/headline_detail/textlink', '1x1', 'textlink',{'apple_daily_article_id':36599626});\n",
      "            halfpage = gpt.createAdSlot('AppleDaily/headline_detail/halfpage', '300x600', 'halfpage');\n",
      "            gpt.enableServices();\n",
      "        })\n",
      "        function gplusoneCallBack(response){\n",
      "          if(response.state=='on'){\n",
      "            ga('send', 'social', 'google_plus', 'share',  '奇蹟 少女漂海21小時獲救');\n",
      "          }\n",
      "        }\n",
      "    </script>, <script type=\"text/javascript\">var _sf_startpt=(new Date()).getTime()</script>, <script type=\"application/ld+json\">\n",
      "    {\n",
      "      \"@context\": \"http://schema.org\",\n",
      "      \"@type\": \"NewsArticle\",\n",
      "      \"headline\": \"奇蹟 少女漂海21小時獲救\",\n",
      "      \"url\": \"http://www.appledaily.com.tw/appledaily/article/headline/20150610/36599626/\",\n",
      "      \"thumbnailUrl\": \"http://twimg.edgesuite.net//images/twapple/640pix/20150610/LN01/LN01_001.jpg\",\n",
      "      \"dateCreated\": \"2015-06-10T06:00Z\",\n",
      "      \"articleSection\": \"蘋果_頭條要聞\",\n",
      "      \"creator\": \"蘋果日報\",\n",
      "      \"keywords\": [\"頭條要聞\"]\n",
      "    }\n",
      "    </script>, <script>\n",
      "        (function(s, p, d) {\n",
      "          var h=d.location.protocol, i=p+\"-\"+s,\n",
      "              e=d.getElementById(i), r=d.getElementById(p+\"-root\"),\n",
      "              u=h===\"https:\"?\"d1z2jf7jlzjs58.cloudfront.net\"\n",
      "              :\"static.\"+p+\".com\";\n",
      "          if (e) return;\n",
      "          e = d.createElement(s); e.id = i; e.async = true;\n",
      "          e.src = h+\"//\"+u+\"/p.js\"; r.appendChild(e);\n",
      "        })(\"script\", \"parsely\", document);\n",
      "        </script>, <script>\r\n",
      "  var _comscore = _comscore || [];\r\n",
      "  _comscore.push({ c1: \"2\", c2: \"8028476\" });\r\n",
      "  (function() {\r\n",
      "    var s = document.createElement(\"script\"), el = document.getElementsByTagName(\"script\")[0]; s.async = true;\r\n",
      "    s.src = (document.location.protocol == \"https:\" ? \"https://sb\" : \"http://b\") + \".scorecardresearch.com/beacon.js\";\r\n",
      "    el.parentNode.insertBefore(s, el);\r\n",
      "  })();\r\n",
      "</script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/1x1/nxm_tr.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/1x1/fingerprint.js\" type=\"text/javascript\"></script>, <script src=\"http://twimg.edgesuite.net/applemobile/js/jquery.cookie.js\" type=\"text/javascript\"></script>, <script>\r\n",
      "    function getUid() {\r\n",
      "        var default_udid = \"0000000000\";\r\n",
      "        var curr_ver = 1;\r\n",
      "        var sig = $.cookie('udid');\r\n",
      "        var ver = $.cookie('udid_ver');\r\n",
      "        if (((sig != undefined) &amp;&amp; (sig != null) &amp;&amp; (sig) &amp;&amp; (sig != \"\")) &amp;&amp; ((ver != undefined) &amp;&amp; (ver != null) &amp;&amp; (ver) &amp;&amp; (ver != \"\") &amp;&amp; (curr_ver &lt;= ver))) {} else {\r\n",
      "            try {\r\n",
      "                var fp = new Fingerprint();\r\n",
      "                sig = fp.get() + '';\r\n",
      "                if (sig.length &lt; 10) {\r\n",
      "                    sig = default_udid + sig;\r\n",
      "                    sig = sig.substr(sig.length - 10);\r\n",
      "                }\r\n",
      "            } catch (err) {\r\n",
      "                sig = default_udid;\r\n",
      "            }\r\n",
      "        }\r\n",
      "            return sig;\r\n",
      "    }\r\n",
      "    var uuid;\r\n",
      "    //cookie 裡面沒有\r\n",
      "    if (typeof($.cookie('uidcookie')) === 'undefined') {\r\n",
      "        //產生新的\r\n",
      "        uuid = getUid();\r\n",
      "        //記到 cookie\r\n",
      "        $.cookie('uidcookie', uuid, {\r\n",
      "                                path: '/',\r\n",
      "                                domain: window.location.hostname.replace(\"www.\", \"\"),\r\n",
      "                                expires: 365 * 100\r\n",
      "                            });\r\n",
      "    }\r\n",
      "    else {\r\n",
      "        uuid=$.cookie('uidcookie');\r\n",
      "    }\r\n",
      "    nxmTrack.nxmAddSeg(\"UID=\"+uuid);\r\n",
      "    nxmTrack.nxmAddSeg(\"REGION=TW\");\r\n",
      "    nxmTrack.nxmAddSeg(\"PROD=ADAILY\");\r\n",
      "    nxmTrack.nxmAddSeg(\"SITE=www.appledaily.com.tw\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"CH=APPLEDAILY\");\r\n",
      "    nxmTrack.nxmAddSeg(\"SECTION=NEWS\");\r\n",
      "    nxmTrack.nxmAddSeg(\"MENU=\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"TITLE=奇蹟 少女漂海21小時獲救\");\r\n",
      "    nxmTrack.nxmAddSeg(\"SUBSECT=headline\");\r\n",
      "    nxmTrack.nxmAddSeg(\"SUBSUBSECT=headline\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"MEDIA=TEXT\");\r\n",
      "    nxmTrack.nxmAddSeg(\"CONTENT=ARTICLE\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"ISSUEID=20150610\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"CID=headline\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"NEWS=DAILY\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"PLATFORM=WEB\"); \r\n",
      "    nxmTrack.nxmAddSeg(\"EDM=MOST\");\r\n",
      "    nxmTrack.nxmAddSeg(\"VERSION=\");\r\n",
      "    nxmTrack.nxmAddSeg(\"DEVICE=\");\r\n",
      "    nxmTrack.nxmAddSeg(\"ACTION=pageview\");\r\n",
      "    nxmTrack.nxmAddSeg(\"DID=\");\r\n",
      "    nxmTrack.nxmAddSeg(\"LAT=\");\r\n",
      "    nxmTrack.nxmAddSeg(\"LON=\");\r\n",
      "</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('door-left');})</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('door-right');})</script>, <script type=\"text/javascript\">\n",
      "                        if (typeof(ONEAD) !== \"undefined\"){\n",
      "                            ONEAD.cmd = ONEAD.cmd || [];\n",
      "                            ONEAD.cmd.push(function(){\n",
      "                                ONEAD_slot('div-incover-ad', 'incover');\n",
      "                                console.log('incover');\n",
      "                            });\n",
      "                        }\n",
      "                    </script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('leaderboard');})</script>, <script type=\"text/javascript\">ONEAD.cmd.push(function(){ ONEAD_slot('div-onead-ad'); });</script>, <script>\n",
      "                                function ShowVideoPlayer() {\n",
      "                                    Player.checkIsiPad();\n",
      "                                    var videoADtagURL = gpt.generateVideoAdTag ('AppleDaily/headline_detail/Preroll', 'Preroll', {'apple_daily_article_id':36599626});\n",
      "                                    var playlist_array = [{url: 'http://twimg.edgesuite.net/video/20150610/9fd9b6d2eb0f77ac7616.mp4'}];\n",
      "                                    setPlayerSize('640', '360');\n",
      "                                    Player.setInitialImage('http://twimg.edgesuite.net/www/extfile/artvideo/20150610/9fd9b6d2eb0f77ac7616_640.jpg')\n",
      "                                    var new_playlist_array = Playlist.adInClip(playlist_array, videoADtagURL);\n",
      "                                    Playlist.setPlayList(new_playlist_array);\n",
      "                                    Player.setStreamingMethod(1);\n",
      "                                }\n",
      "                                function liveStateChanged(stateMessage) {\n",
      "                                    if (stateMessage.indexOf('playerviewthrough')!=-1) {\n",
      "                                        var viewThrough = stateMessage.split(\"-\")[1].split(\",\");\n",
      "                                        var viewPercent = 'View '+viewThrough[0];\n",
      "                                        ga('send', 'event', 'video', viewPercent, '日報/頭條要聞/20150610 奇蹟 少女漂海21小時獲救');\n",
      "                                    }                                                                    \n",
      "                                    if ((stateMessage=='onStart')) {\n",
      "                                        $.ajax({\n",
      "                                             url:'http://weblogging.appledaily.com.tw/api/SetClick/n/a/36599626/5/1077/20150610/appledaily/'+Math.random(),\n",
      "                                             dataType: 'jsonp',\n",
      "                                             success: function(response) {}\n",
      "                                        });\n",
      "                                    }\n",
      "                                    if ((stateMessage=='onFinish')) {\n",
      "                                        if (document.getElementsByClassName(\"aftervideo\").length) document.getElementsByClassName(\"aftervideo\")[0].style['display']='block';\n",
      "                                        if (/(iphone|ipad|pod)/ig.test(navigator.platform)) $($f(\"flow_player\").getParent()).hide();\n",
      "                                    }\n",
      "                                    \n",
      "                                }\n",
      "                            </script>, <script>\n",
      "                                        ShowVideoPlayer();\n",
      "                                        Player.create('videobox');\n",
      "                                        Player.open();\n",
      "                                        chartbeatvideo();\n",
      "                                    </script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/jquery.appledaily.html5video.js\" type=\"text/javascript\"></script>, <script>\n",
      "                                    var isAndroid = /(android)/ig.test(navigator.appVersion);\n",
      "                                    if (isAndroid) {\n",
      "                                      $(document).ready(function() {\n",
      "                                        $('#videobox').html5video({\n",
      "                                          'src': 'http://twimg.edgesuite.net/video/20150610/9fd9b6d2eb0f77ac7616.mp4',\n",
      "                                          'poster': 'http://twimg.edgesuite.net/www/extfile/artvideo/20150610/9fd9b6d2eb0f77ac7616_640.jpg',\n",
      "                                          'ageCheck': ($('#popup_18').length &gt; 0),\n",
      "                                          'btnAgeCheckYes': $('#newimglayer a.yes')\n",
      "                                        }).on('onStart', function(e) {\n",
      "                                          $.ajax({\n",
      "                                               url:'http://weblogging.appledaily.com.tw/api/SetClick/n/a/36599626/5/1077/20150610/appledaily/'+Math.random(),\n",
      "                                               dataType: 'jsonp',\n",
      "                                               success: function(response) {}\n",
      "                                          });\n",
      "                                        }).on('onFinish', function(e) {\n",
      "                                         $('.aftervideo').show();\n",
      "                                        }).on('onCuePoint', function (e, data) {\n",
      "                                          if (window.hasOwnProperty('liveStateChanged') &amp;&amp; (typeof window.liveStateChanged == 'function')) {\n",
      "                                            liveStateChanged(data.percentStr);\n",
      "                                          }\n",
      "                                        }).on('onAgeCheckPass', function(e) {\n",
      "                                          $('#newimglayer').hide();\n",
      "                                          $('#unImage').hide();\n",
      "                                          chartbeatvideo();\n",
      "                                        });\n",
      "                                      });                               \n",
      "                                    }\n",
      "                                </script>, <script async=\"\" defer=\"\" src=\"//assets.pinterest.com/js/pinit.js\" type=\"text/javascript\"></script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display(\"textlink\");})</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('articlelast');})</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('articlelast2');})</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('articlelast3');})</script>, <script>\n",
      "                            function changAnimation(count) {\n",
      "                                $.ajax({\n",
      "                                        url: \"/appledaily/videoajax/\"+count+\"/\"+Math.random(),\n",
      "                                        dataType: 'html',\n",
      "                                        error: function(xhr){},\n",
      "                                        success: function(response) {$('#videolist').html(response);}\n",
      "                                  });\n",
      "                            }\n",
      "                            changAnimation(0);\n",
      "                        </script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('rectangleAD1');})</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display('rectangleAD2');})</script>, <script>$(function() {displayFBlike(1);});</script>, <script language=\"javascript\">\r\n",
      "            (function(d, s, id) {\r\n",
      "              var js, fjs = d.getElementsByTagName(s)[0];\r\n",
      "              if (d.getElementById(id)) return;\r\n",
      "              js = d.createElement(s); js.id = id;\r\n",
      "              js.src = \"//connect.facebook.net/zh_TW/all.js#xfbml=1\";\r\n",
      "              fjs.parentNode.insertBefore(js, fjs);\r\n",
      "            }(document, 'script', 'facebook-jssdk'));\r\n",
      "        </script>, <script>\r\n",
      "    function displayFBlike(option) {    \t\r\n",
      "        if (option==1) {\r\n",
      "            $('#fbifm').remove();if($('.fb-like-box').is(':hidden')){$('.fb-like-box').show();}\r\n",
      "        }\r\n",
      "        if (option==2) {\r\n",
      "            $('.fb-like-box').hide();\r\n",
      "            if ($('#fbifm').length==0) {\r\n",
      "                $('#changeFBlike').append('&lt;iframe id=\"fbifm\" src=\"http://www.facebook.com/plugins/activity.php?site=www.appledaily.com.tw&amp;action&amp;width=300&amp;height=370&amp;header=true&amp;colorscheme=light&amp;linktarget=_blank&amp;border_color&amp;font&amp;recommendations=true&amp;\" scrolling=\"yes\" frameborder=\"0\" style=\"border:none; overflow:hidden; width:300px; height:370px;\" allowTransparency=\"true\"&gt;&lt;/iframe&gt;');\r\n",
      "            }\r\n",
      "        }\r\n",
      "    }\r\n",
      "</script>, <script type=\"text/javascript\">googletag.cmd.push(function() {googletag.display(\"halfpage\");})</script>, <script>\n",
      "                dataLayer = [{\n",
      "                'pageLevel' : 'Article', \n",
      "                'videoArticle': 'video', \n",
      "                'publishDay': 'Wednesday', \n",
      "                'columnist' : '', \n",
      "                'column' : ''\n",
      "                }];\n",
      "            </script>, <script src=\"http://twimg.edgesuite.net/appledaily/images/js/jquery.appledaily.backtotop.js\"></script>, <script>\n",
      "    $(\".pushbtn\").click(function () {\n",
      "    if($('#subSection').css('display')=='none'){\n",
      "       $('#subSection').css('display','block');\n",
      "       $('#subSection2').css('display','none');\n",
      "       $('#subSection3').css('display','none');\n",
      "    }\n",
      "    else{\n",
      "       $('#subSection').css('display','none');\n",
      "    }\n",
      "    });\n",
      "    $(\".pushbtn2\").click(function () {\n",
      "    if($('#subSection2').css('display')=='none'){\n",
      "       $('#subSection2').css('display','block');\n",
      "       $('#subSection').css('display','none');\n",
      "       $('#subSection3').css('display','none');\n",
      "    }\n",
      "    else{\n",
      "       $('#subSection2').css('display','none');\n",
      "    }\n",
      "    });\n",
      "    $(\".pushbtn3\").click(function () {\n",
      "    if($('#subSection3').css('display')=='none'){\n",
      "       $('#subSection3').css('display','block');\n",
      "       $('#subSection').css('display','none');\n",
      "       $('#subSection2').css('display','none');\n",
      "    }\n",
      "    else{\n",
      "       $('#subSection3').css('display','none');\n",
      "    }\n",
      "    });\n",
      "\n",
      "    $(document).ready(function() {\n",
      "        var opacity = 0.5,\n",
      "            toOpacity = 1,\n",
      "            duration = 250;\n",
      "        $('.opacity').css('opacity', opacity).hover(function() {\n",
      "            $(this).fadeTo(duration, toOpacity);\n",
      "        }, function() {\n",
      "            $(this).fadeTo(duration, opacity);\n",
      "        });\n",
      "        \n",
      "        $.ajax({\n",
      "                    url: '/index/getpv/',\n",
      "                    success: function(response){\n",
      "                        $('#pv').html(response);\n",
      "                    }\n",
      "                });\n",
      "\n",
      "        $(window).scroll(function() {\n",
      "          if($(this).scrollTop()&gt;130&amp;&amp;($(this).scrollTop()-(-800)&lt;$(document).height()-350)){\n",
      "              $('#door').css('position','fixed').css('top','-45px');\n",
      "          }else{\n",
      "              $('#door').css('position','relative').css('top','0px');\n",
      "          }\n",
      "        });\n",
      "\n",
      "        $.tools.dateinput.localize(\"tw\", {\n",
      "            months:       '一月,二月,三月,四月,五月,六月,七月,八月,九月,十月,十一月,十二月',\n",
      "            shortMonths:  '一,二,三,四,五,六,七,八,九,十,十一,十二',\n",
      "            days:         '日,一,二,三,四,五,六',\n",
      "            shortDays:    '日,一,二,三,四,五,六'\n",
      "        });\n",
      "\n",
      "        $(\"#DailyDate\").dateinput({lang: 'tw',selectors: true,format: 'yyyymmdd',trigger: true ,min:'2003-05-02',max:'2015-06-14',offset: [0,-50],speed: 'fast',yearRange:[-20,1]});\n",
      "        $(\"#DailyDate\").change(function() {location.href=\"/appledaily/archive/\"+$(\"#DailyDate\").val();});\n",
      "        /*$(\".nbtn a.caltrigger\").append(\"昔日新聞\");*/\n",
      "        $(\".nbtn a.caltrigger\").append(\"&lt;img src='http://twimg.edgesuite.net/appledaily/images/core/calendar_icon.png'/&gt;\");\n",
      "    });\n",
      "  </script>, <script>\n",
      "  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){\n",
      "      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),\n",
      "      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)\n",
      "  })(window,document,'script','//www.google-analytics.com/analytics.js','ga');\n",
      "\n",
      "  ga('create', 'UA-2067247-40', 'auto', {'sampleRate':1});\n",
      "  ga('require', 'linkid', 'linkid.js');\n",
      "  ga('require', 'displayfeatures');\n",
      "    \n",
      "</script>, <script>\n",
      "     \n",
      "    (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':\n",
      "    new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],\n",
      "    j=d.createElement(s),dl=l!='dataLayer'?'&amp;l='+l:'';j.async=true;j.src='//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);\n",
      "    })(window,document,'script','dataLayer','GTM-P5MXMC');\n",
      "</script>, <script type=\"text/javascript\">\n",
      "    (function(w, d) { var a = function() { var b = d.createElement('script'); b.type = 'text/javascript';\n",
      "    if (undefined !== b.setAttribute) { b.setAttribute('async', 'async'); }\n",
      "    b.src = '//' + ((w.location.protocol === 'https:') ? 's3.amazonaws.com/cdx-radar/' :\n",
      "      'radar.cedexis.com/') + '01-14272-radar10.min.js'; d.body.appendChild(b); };\n",
      "    if (w.addEventListener) { w.addEventListener('load', a, false); }\n",
      "    else if (w.attachEvent) { w.attachEvent('onload', a); }\n",
      "    }(window, document));\n",
      "</script>, <script type=\"text/javascript\">\n",
      "        ga('ga_gtm.set', 'dimension11', '');ga('ga_gtm.set', 'dimension10', '');ga('ga_gtm.set', 'dimension9', 'Wednesday');ga('ga_gtm.set', 'dimension6', 'video');ga('ga_gtm.set', 'dimension8', 'appledaily daily');        // 圖片 PV\n",
      "        function setimgpv(){\n",
      "            $.ajaxSetup({ cache: false });\n",
      "            $.ajax({\n",
      "                  url: 'http://weblogging.appledaily.com.tw/api/SetClick/n/n/36599626/5/1077/20150610/appledaily/'+Math.random(),\n",
      "                  dataType: 'jsonp',\n",
      "                  success: function(data) {\n",
      "                  }\n",
      "              })\n",
      "        }\n",
      "\n",
      "      //更改字級大小\n",
      "        function doZoom(size) {\n",
      "            size=size*1;\n",
      "            $('#smaller').removeClass();\n",
      "            $('#middle').removeClass();\n",
      "            $('#bigger').removeClass();\n",
      "            $('#smaller').addClass('smaller overcooked');\n",
      "            $('#middle').addClass('middle overcooked');\n",
      "            $('#bigger').addClass('bigger overcooked');\n",
      "            switch(size){\n",
      "              case 16:\n",
      "                $('#smaller').addClass('smon');\n",
      "                break;\n",
      "              case 18:\n",
      "                $('#middle').addClass('mion');\n",
      "                break;\n",
      "              case 20:\n",
      "                $('#bigger').addClass('bion');\n",
      "                break;\n",
      "            }\n",
      "            document.cookie=\"FRONTSIZE=\"+size+\"; path=/\";\n",
      "            $('#h1').css({'font-size': (size+8)+'px'});\n",
      "            $('#h2').css({'font-size': (size+2)+'px'});\n",
      "            $('#introid').css({'font-size': size+'px'});\n",
      "            $('h2#bhead').css({'font-size': size+'px'});\n",
      "            $('p#bcontent').css({'font-size': size+'px'});\n",
      "        }\n",
      "\n",
      "        function getCookieData(name) {\n",
      "            var patrn = new RegExp( \"^\" + name + \"=(.*?);\" ),\n",
      "                patr2 = new RegExp( \" \" + name + \"=(.*?);\" );\n",
      "            if ( match = (document.cookie.match(patrn) || document.cookie.match(patr2)) )\n",
      "               return match[1];\n",
      "            return false;\n",
      "        }\n",
      "\n",
      "        $(document).ready(function() {\n",
      "            fontsize = getCookieData('FRONTSIZE');\n",
      "            if(fontsize) {\n",
      "                  doZoom(fontsize);\n",
      "              }\n",
      "              else {\n",
      "                  doZoom(18);\n",
      "              }\n",
      "              $('.lbimg a').lightBox();\n",
      "              $.ajax({\n",
      "                url: 'http://weblogging.appledaily.com.tw/api/SetClick/n/v/36599626/5/1077/20150610/appledaily/'+Math.random(),\n",
      "                dataType: 'jsonp',\n",
      "                success: function(data) {\n",
      "                }\n",
      "              });\n",
      "        });\n",
      "        window.___gcfg = {lang: 'zh-TW'};\n",
      "        (function() {\n",
      "          var po = document.createElement('script'); po.type = 'text/javascript'; po.async = true;\n",
      "          po.src = 'https://apis.google.com/js/plusone.js';\n",
      "          var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(po, s);\n",
      "        })();\n",
      "        \n",
      "        function MM_jumpMenu(targ,selObj,restore){\n",
      "            eval(targ+\".location='\"+selObj.options[selObj.selectedIndex].value+\"'\");\t\t\n",
      "            if (restore) selObj.selectedIndex=0;  \n",
      "        }\n",
      "    </script>, <script type=\"text/javascript\">\n",
      "        var _sf_async_config={};\n",
      "        /** CONFIGURATION START **/\n",
      "        _sf_async_config.uid = 49443;\n",
      "        _sf_async_config.domain = 'appledaily.com.tw';\n",
      "        _sf_async_config.useCanonical = true;\n",
      "        _sf_async_config.sections = '蘋果日報_頭條要聞';\n",
      "        _sf_async_config.authors = '蘋果日報';\n",
      "        _sf_async_config.autoDetect = false;\n",
      "        /** CONFIGURATION END **/\n",
      "        (function(){\n",
      "            function loadChartbeat() {\n",
      "            //console.log(\"----------loadChartbeat----------\");\n",
      "                window._sf_endpt=(new Date()).getTime();\n",
      "                var e = document.createElement('script');\n",
      "                e.setAttribute('language', 'javascript');\n",
      "                e.setAttribute('type', 'text/javascript');\n",
      "                                    e.setAttribute('src','//static.chartbeat.com/js/chartbeat_video.js');\n",
      "                                document.body.appendChild(e);\n",
      "            }\n",
      "            var oldonload = window.onload;\n",
      "            window.onload = (typeof window.onload != 'function') ?\n",
      "            loadChartbeat : function() { oldonload(); loadChartbeat();};\n",
      "        })();\n",
      "    </script>, <script>\n",
      "        function page_do(artid,issueid,page,app_name){    \n",
      "              $.ajax({\n",
      "                      url: '/appledaily/getforum/artid/'+artid+'/issueid/'+issueid+'/page/'+page+'/app_name/'+app_name,                                                    \n",
      "                      dataType: 'html',\n",
      "                      type:\"GET\",\n",
      "                      error: function(xhr) {      },\n",
      "                      success: function(response) {          \n",
      "                          $('#forum_list').html(response);\n",
      "                      }\n",
      "                });\n",
      "        }                                       \n",
      "\n",
      "        function post_show(typename){\t\n",
      "          if($('#post_username').val().length == 0  || $('#post_username').val().length &gt; 15){\n",
      "              alert('暱稱限制在 1- 15字以內!!');\n",
      "              return false;\n",
      "          }\t\n",
      "          if($('#post_text').val().length == 0 || $('#post_text').val()=='留言' || $('#post_text').val().length &gt; 140){\n",
      "              alert('留言字數限制在 1- 140字以內!!');\n",
      "              return false;\n",
      "          }\t\t \n",
      "          $.ajax({      \n",
      "              url: '/appledaily/postforum/type/article/'+Math.random(),    \n",
      "              type: 'POST',      \n",
      "                  data: {        \n",
      "                      post_username: $('#post_username').val(),       \n",
      "                      post_text: $('#post_text').val(),\n",
      "                      artid: '36599626',\n",
      "                      issueid: '20150610',\n",
      "                      app_name: 'twapple',    \n",
      "                      parsecid: '5',\n",
      "                      secid: '1077',\n",
      "                      CHK: 'e25c096c0b7cf3dc9d8cacacabb1521f'\n",
      "                  },\n",
      "                  error: function(xhr) {\n",
      "                      alert('發送錯誤');\n",
      "                  },                                                       \n",
      "                  success: function(response) {\t  \n",
      "                  switch(response){\t  \n",
      "                      case '1':\n",
      "                          alert('為防止洗版影響網友閱讀，您暫時無法留言。');                                                                                       \t\t  \t\n",
      "                      break;\n",
      "                      case '2':\n",
      "                          alert('因您的留言違反討論區規範，無法留言。');\n",
      "                      break;\n",
      "                      case '3':\n",
      "                          alert('為防止洗版影響網友閱讀權益，目前您無法留言。');  \t\n",
      "                      break;\n",
      "                      default:\n",
      "                          alert('發送成功');\n",
      "                      if(typename=='article'){    \n",
      "                          $('#forum_list').html(response);\n",
      "                          $('#forum_list').fadeIn();\n",
      "                      }    \n",
      "                      if(typename=='amn'){    \t  \n",
      "                          $('#forumbox').html(response);\n",
      "                          $('#forumbox').fadeIn();\n",
      "                      }    \n",
      "                      document.postforum.post_username.value='';\n",
      "                      document.postforum.post_text.value=''; \n",
      "                      break;\n",
      "                  }\n",
      "              }\n",
      "          });   \n",
      "        }\n",
      "    </script>]\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import Select\n",
    "\n",
    "driver = webdriver.Firefox()\n",
    "driver.get(\"http://www.appledaily.com.tw/appledaily/article/headline/20150610/36599626/\")\n",
    "html = driver.page_source\n",
    "content = str(html.encode('utf-8'))\n",
    "soup = BeautifulSoup(content)\n",
    "print soup.select('.articulum')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "記者賴映秀\n",
      "賴映秀\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "c =\"<p>記者賴映秀／台北報導</p>  <p>記者蘇晟彥／綜合報導</p>  <p>文／賴映秀</p>   <p>圖／和和</p>\"\n",
    "try:\n",
    "    print re.search('<p>([^圖].*?)／.*?報導<',c).group(1)\n",
    "except:\n",
    "    pass\n",
    "try:\n",
    "    print re.search('>文／(.*?)<',c).group(1)    #print re.search('>[文]／(.*?)<',c).group(1)\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'console' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-9bdb73cd719c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mconsole\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"a\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'console' is not defined"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}