0_Data_Extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# https://marcobonzanini.com/2015/03/09/mining-twitter-data-with-python-part-2/\n",
    "\n",
    "# Libreria teewpy\n",
    "\n",
    "import os\n",
    "import tweepy\n",
    "import sys\n",
    "from tweepy.streaming import StreamListener\n",
    "from tweepy import Stream\n",
    "\n",
    "config = {} \n",
    "config_path = os.path.join(os.path.abspath('..')) \n",
    "config_name = r'config.py' \n",
    "config_file = os.path.join(config_path,config_name) \n",
    "exec(open(config_file).read(),config)\n",
    "\n",
    "\n",
    "# Key and Secret\n",
    "consumer_key=config['TWITTER_KEY']\n",
    "consumer_secret=config['TWITTER_SECRET']\n",
    "access_token=config['TOKEN']\n",
    "access_token_secret=config['TOKEN_SECRET']\n",
    "\n",
    "#HashTag to Check\n",
    "hashtag = 'sundayevening'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Authentication\n",
    "\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_token_secret)\n",
    "\n",
    "api = tweepy.API(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Data Extraction - Stream\n",
    "# https://dev.twitter.com/streaming/overview\n",
    "\n",
    "class MyListener(StreamListener):\n",
    "    \n",
    "    tweet_number=0   # class variable\n",
    "    \n",
    "    def __init__(self,max_tweets):\n",
    "        self.max_tweets=max_tweets # max number of tweets\n",
    " \n",
    "    def on_data(self, data):\n",
    "        self.tweet_number+=1 \n",
    "        try:\n",
    "            with open(hashtag+'.json', 'a') as f:\n",
    "                f.write(data)\n",
    "        except BaseException as e:\n",
    "            print('Error on_data: %s' % str(e))\n",
    "            \n",
    "        if self.tweet_number>=self.max_tweets:\n",
    "            sys.exit('Limit of '+str(self.max_tweets)+' tweets reached.')\n",
    " \n",
    "    def on_error(self, status):\n",
    "        print(status)\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "420\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-4fab579badf0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Read Stream of tweets based on HashTag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtwitter_stream\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mStream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mauth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMyListener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtwitter_stream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'#'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mhashtag\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlanguages\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'en'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/cesar/anaconda/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36mfilter\u001b[0;34m(self, follow, track, async, locations, stall_warnings, languages, encoding, filter_level)\u001b[0m\n\u001b[1;32m    443\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'delimited'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'length'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    444\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'stream.twitter.com'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 445\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m     def sitestream(self, follow, stall_warnings=False,\n",
      "\u001b[0;32m/Users/cesar/anaconda/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36m_start\u001b[0;34m(self, async)\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 361\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    363\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cesar/anaconda/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    253\u001b[0m                         self.retry_time = max(self.retry_420_start,\n\u001b[1;32m    254\u001b[0m                                               self.retry_time)\n\u001b[0;32m--> 255\u001b[0;31m                     \u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretry_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    256\u001b[0m                     self.retry_time = min(self.retry_time * 2,\n\u001b[1;32m    257\u001b[0m                                           self.retry_time_cap)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Read Stream of tweets based on HashTag\n",
    "twitter_stream = Stream(auth, MyListener(10000))\n",
    "twitter_stream.filter(track=['#'+hashtag],languages=['en'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}