ft.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Sentence extractor for The Financial Times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import math\n",
    "import os\n",
    "from collections import defaultdict\n",
    "from datetime import timedelta, date, datetime\n",
    "from dateutil import parser\n",
    "from time import sleep, time\n",
    "\n",
    "import requests\n",
    "import yaml\n",
    "from joblib import Parallel, delayed\n",
    "from pymongo import MongoClient\n",
    "from pymongo.errors import BulkWriteError, DuplicateKeyError"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.remove('search_ft.log')\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logging.getLogger().handlers = []\n",
    "logging.getLogger('requests.packages.urllib3').setLevel(logging.WARNING)\n",
    "logging.basicConfig(filename='search_ft.log', level=logging.INFO, format='%(asctime)s %(message)s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def write_log(*args, status=None):\n",
    "    record = '{} ==> {}'.format(args, status)\n",
    "    logging.info(record)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##MongoDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "client = MongoClient()\n",
    "client.drop_database('ft')\n",
    "db = client.ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def update_document(_id, new_q_info):\n",
    "    doc = db.articles.find_one({'_id': _id})\n",
    "    new_orig_term = list(new_q_info.keys())[0]\n",
    "    if new_orig_term in doc['q_info']:\n",
    "        doc['q_info'][new_orig_term].extend(new_q_info[new_orig_term])\n",
    "    else:\n",
    "        doc['q_info'].update(new_q_info)\n",
    "    db.articles.update({'_id': _id}, {'$set': {'q_info': doc['q_info']}})\n",
    "\n",
    "def insert_documents(docs, q):\n",
    "    try:\n",
    "        inserted = db.articles.insert_many(docs, ordered=False)\n",
    "        total_inserted = len(inserted.inserted_ids)\n",
    "        write_log(q, status='INSERTION OK {}'.format(total_inserted))\n",
    "    except BulkWriteError as e:\n",
    "        n_updated = 0\n",
    "        for err in e.details['writeErrors']:\n",
    "            if err['code'] == 11000:\n",
    "                try:\n",
    "                    _id = err['op']['_id']\n",
    "                    new_q_info = err['op']['q_info']\n",
    "                    update_document(_id, new_q_info)\n",
    "                    n_updated += 1\n",
    "                except Exception as ex:\n",
    "                    write_log(q, status='UPDATE EXCEPTION {} - {}'.format(err['errmsg'], ex))\n",
    "        write_log(q, status='INSERTION OK {}'.format(e.details['nInserted']))\n",
    "        write_log(q, status='UPDATE OK {}'.format(n_updated))\n",
    "    except Exception as e:\n",
    "        write_log(q, status='INSERTION EXCEPTION {}'.format(e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Search terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def preprocess_term(term):\n",
    "    terms = []\n",
    "    \n",
    "    curated_term = term.lower()\n",
    "    curated_term = curated_term.replace(' & ', ' ')\n",
    "    curated_term = curated_term.replace(' and ', ' ')\n",
    "    curated_term = curated_term.replace('-', ' ')\n",
    "    terms.append(curated_term)\n",
    "    \n",
    "    for term in terms:\n",
    "        if term.endswith('corporation'):\n",
    "            terms.append(term[:-12])\n",
    "        elif term.endswith('corp'):\n",
    "            terms.append(term[:-5])\n",
    "        elif term.endswith('company'):\n",
    "            terms.append(term[:-8])\n",
    "        elif term.endswith('inc.'):\n",
    "            terms.append(term[:-5])\n",
    "        elif term.endswith('inc'):\n",
    "            terms.append(term[:-4])\n",
    "        elif term.endswith('.com'):\n",
    "            terms.append(term[:-4])\n",
    "    \n",
    "    return terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "term_dict = {}\n",
    "for i in range(1, 4):\n",
    "    term_dict[i] = {}\n",
    "    st_file = open('search_terms_{}.txt'.format(i))\n",
    "    term_list = map(lambda x: x.strip(), st_file.readlines())\n",
    "    for term in term_list:\n",
    "        term_dict[i][term] = preprocess_term(term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('search_terms_ft.yml', 'w') as search_term_file:\n",
    "    search_term_file.write(yaml.dump(term_dict, default_flow_style=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('search_terms_ft.yml') as search_term_file:\n",
    "    term_yaml = yaml.load(search_term_file.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##FT API keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# One API key for each of the cores\n",
    "api_keys = [\n",
    "    \"mtqjgqpd63c5aky6ujz7mfe4\", # Antonio\n",
    "    \"zb3atmjyfth2ehgzbr94nkq2\", # Antonio 2\n",
    "    \"d67kd552dhk4spqhjnx2kzz8\", # Javi\n",
    "    \"x6f2ydhhyuv9exjresaxq347\", # David\n",
    "    \"gtya39yhnt3uz558s9hf8aqs\", # Gabi\n",
    "    \"85uppscwzvjbx8w8bekvfs2k\", # Nandi\n",
    "    \"mf5k3xz7v7f66jcx6yx7g2q3\", # Adri\n",
    "    \"warj9jjwsuayfqajgggf7ac8\", # Pedro\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def next_multiple(n, m):\n",
    "    # 4, 17 ==> 20\n",
    "    return math.ceil(m / n) * n\n",
    "\n",
    "def chunks(l, n_chunks):\n",
    "    l = list(l)\n",
    "    size = len(l)\n",
    "    n = next_multiple(n_chunks, size) // n_chunks\n",
    "    for i in range(0, size, n):\n",
    "        yield l[i:i + n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# [(search_term, original_term, term_type)]\n",
    "search_terms = [(t, k2, k1)\n",
    "                   for k1 in term_yaml\n",
    "                       for k2 in term_yaml[k1]\n",
    "                           for t in term_yaml[k1][k2]\n",
    "              ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TEST\n",
    "\n",
    "# api_keys = [\n",
    "#     \"mf5k3xz7v7f66jcx6yx7g2q3\", # Adri\n",
    "#     \"warj9jjwsuayfqajgggf7ac8\", # Pedro\n",
    "# ]\n",
    "\n",
    "# search_terms = [\n",
    "#     ('entrepreneur', 'entrepreneur', 1),\n",
    "#     ('executive', 'executive', 1),\n",
    "#     ('google', 'google', 2),\n",
    "#     ('amazon', 'amazon.com', 2),\n",
    "#     ('amazon.com', 'amazon.com', 3),\n",
    "#     ('ebay', 'e-bay', 3),\n",
    "# ]\n",
    "\n",
    "# BEGIN_DATE = date(2014, 1, 1)\n",
    "# END_DATE = date(2014, 2, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "search_terms_by_api_key = defaultdict(list)\n",
    "for i in range(1, 4):\n",
    "    search_terms_i = list(filter(lambda x: x[2] == i, search_terms))\n",
    "    for t in zip(api_keys, chunks(search_terms_i, len(api_keys))):\n",
    "        search_terms_by_api_key[t[0]].extend(t[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def month_duration(d):\n",
    "    if d.month in [1, 3, 5, 7, 8, 10, 12]:\n",
    "        ndays = 31\n",
    "    elif d.month in [4, 6, 9, 11]:\n",
    "        ndays = 30\n",
    "    else: # d.month == 2\n",
    "        if d.year % 400 == 0 or d.year % 4 == 0 and d.year % 100 != 0: # lap-year\n",
    "            ndays = 29\n",
    "        else:\n",
    "            ndays = 28\n",
    "    return ndays\n",
    "\n",
    "def n_days(d, n_months):\n",
    "    ndays = 0\n",
    "    new_d = d\n",
    "    for _ in range(n_months):\n",
    "        m_duration = month_duration(d)\n",
    "        d += timedelta(m_duration)\n",
    "        ndays += m_duration\n",
    "    return ndays - 1\n",
    "\n",
    "def date_ranges(begin_date, end_date, n_months=1):\n",
    "    aux_date = begin_date\n",
    "    while aux_date < end_date:\n",
    "        ndays = n_days(aux_date, n_months)\n",
    "        yield (aux_date, min(aux_date + timedelta(ndays + 1), end_date))\n",
    "        aux_date += timedelta(ndays + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Timer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LAST_REQUEST = time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def wait(f, *args, t=9):\n",
    "    global LAST_REQUEST\n",
    "    now = time()\n",
    "    elapsed_time = now - LAST_REQUEST\n",
    "    if elapsed_time < t:\n",
    "        sleep(t - elapsed_time)\n",
    "    LAST_REQUEST = time()\n",
    "    return f(*args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Query:\n",
    "    def __init__(self, term, begin_date, end_date, offset, api_key):\n",
    "        self.term = term\n",
    "        self.begin_date = begin_date\n",
    "        self.end_date = end_date\n",
    "        self.offset = offset\n",
    "        self.api_key =api_key\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return 'Q<{}, {}, {}, {}, {}>'.format(self.term, self.begin_date, self.end_date, self.offset, self.api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Downloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BEGIN_DATE = date(1999, 1, 1)\n",
    "END_DATE = date(2014, 12, 31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def search(q):\n",
    "    try:\n",
    "        base_url = 'http://api.ft.com/content/search/v1'\n",
    "        payload = {'apiKey': q.api_key}\n",
    "        data = {\n",
    "            'queryString': '{} AND initialPublishDateTime:>{} AND initialPublishDateTime:<{}'.format(\n",
    "                q.term,\n",
    "                q.begin_date + 'T00:00:00Z',\n",
    "                q.end_date + 'T00:00:00Z',\n",
    "            ),\n",
    "            'resultContext': {\n",
    "                'aspects': [\"title\",\"lifecycle\",\"location\",\"summary\",\"editorial\"],\n",
    "                'offset': q.offset,\n",
    "                'sortOrder' : 'ASC',\n",
    "                'sortField' : 'initialPublishDateTime'\n",
    "            }\n",
    "        }\n",
    "        r = requests.post(base_url, params=payload, json=data)\n",
    "        response = r.json()\n",
    "        response['status'] = r.reason\n",
    "        \n",
    "        if response['results']:\n",
    "            write_log(q, status='SEARCH OK {}'.format(response['results'][0]['indexCount']))\n",
    "        else:\n",
    "            write_log(q, status='SEARCH ERROR')\n",
    "    except ValueError as e:\n",
    "        if str(e) == 'Expecting value: line 1 column 1 (char 0)':\n",
    "            write_log(q, status='SEARCH ERROR api-key')\n",
    "        else:\n",
    "            write_log(q, status='SEARCH EXCEPTION {}'.format(e))\n",
    "        response = {'status': r.reason}\n",
    "    except Exception as e:\n",
    "        write_log(q, status='SEARCH EXCEPTION {}'.format(e))\n",
    "        response = {'status': r.reason}\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# base_url = 'http://api.ft.com/content/search/v1'\n",
    "# payload = {'apiKey': \"mtqjgqpd63c5aky6ujz7mfe4\"}\n",
    "# data = {\n",
    "#     'queryString': '{} AND initialPublishDateTime:>{} AND initialPublishDateTime:<{}'.format(\n",
    "#         'executive',\n",
    "#         \"2014-01-01\" + 'T00:00:00Z',\n",
    "#         \"2014-01-07\" + 'T00:00:00Z',\n",
    "#     ),\n",
    "#     'resultContext': {\n",
    "#         'aspects': [\"title\",\"lifecycle\",\"location\",\"summary\",\"editorial\"],\n",
    "#         'offset': 0,\n",
    "#         'sortOrder' : 'ASC',\n",
    "#         'sortField' : 'initialPublishDateTime'\n",
    "#     }\n",
    "# }\n",
    "# r = requests.post(base_url, params=payload, json=data)\n",
    "# response = r.json()\n",
    "# response['status'] = r.reason"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_documents_by_offset(q, term, total_results):\n",
    "    n_pages = math.ceil(total_results / 100)\n",
    "    for page in range(n_pages):\n",
    "        q.offset = page * 100\n",
    "        response = wait(search, q)\n",
    "        \n",
    "        if response['status'] == 'OK':\n",
    "            docs = response['results'][0]['results']\n",
    "            for doc in docs:\n",
    "                doc['_id'] = doc.pop('id')\n",
    "                orig_term = term[1].replace('.', '_')\n",
    "                doc.update({'q_info': {orig_term: [{'q': q.__dict__, 'term_category': term[2]}]}})\n",
    "                # TODO: filter fields ... DO NOT FILTER\n",
    "            insert_documents(docs, q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_documents_by_query(q, term):\n",
    "    response = wait(search, q)\n",
    "    \n",
    "    if response['status'] == 'OK':\n",
    "        total_results = response['results'][0]['indexCount']\n",
    "        if total_results <= 4000:\n",
    "            get_documents_by_offset(q, term, total_results)\n",
    "        else:\n",
    "            bd = parser.parse(q.begin_date)\n",
    "            ed = parser.parse(q.end_date)\n",
    "            half = (ed - bd) // 2\n",
    "            \n",
    "            begin_date1 = q.begin_date\n",
    "            end_date1 = (bd + timedelta(half.days)).strftime(\"%Y-%m-%d\")\n",
    "            q1 = Query(q.term, begin_date1, end_date1, 0, q.api_key)\n",
    "            get_documents_by_query(q1, term)\n",
    "            \n",
    "            begin_date2 = (bd + timedelta(half.days + 1)).strftime(\"%Y-%m-%d\")\n",
    "            end_date2 = q.end_date\n",
    "            q2 = Query(q.term, begin_date2, end_date2, 0, q.api_key)\n",
    "            get_documents_by_query(q2, term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_by_date_ranges(term, api_key):\n",
    "    for r in date_ranges(BEGIN_DATE, END_DATE, 1):\n",
    "        begin_date = r[0].strftime(\"%Y-%m-%d\")\n",
    "        end_date = r[1].strftime(\"%Y-%m-%d\")\n",
    "        q = Query(term[0], begin_date, end_date, 0, api_key)\n",
    "        get_documents_by_query(q, term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def download_by_term(term, api_key):\n",
    "    begin_date = BEGIN_DATE.strftime(\"%Y-%m-%d\")\n",
    "    end_date = END_DATE.strftime(\"%Y-%m-%d\")\n",
    "    q = Query(term[0], begin_date, end_date, 0, api_key)\n",
    "    response = wait(search, q)\n",
    "    \n",
    "    if response['status'] == 'OK':\n",
    "        total_results = response['results'][0]['indexCount']\n",
    "        if total_results <= 4000:\n",
    "            get_documents_by_offset(q, term, total_results)\n",
    "        else:\n",
    "            download_by_date_ranges(term, api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_by_key(terms, api_key):\n",
    "    for term in terms:\n",
    "        try:\n",
    "            download_by_term(term, api_key)\n",
    "        except Exception as e:\n",
    "            write_log(e, status='DOWNLOAD EXCEPTION {} {}'.format(term, api_key))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def downloader(search_terms_by_api_key, api_keys):\n",
    "    # Version parallel\n",
    "    Parallel(n_jobs=8)(delayed(download_by_key)(search_terms_by_api_key[api_key], api_key) for api_key in api_keys)\n",
    "\n",
    "    # Version sequencial\n",
    "#     for api_key in api_keys:\n",
    "#         download_by_key(search_terms_by_api_key[api_key], api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "downloader(search_terms_by_api_key, api_keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "summary : {'excerpt': 'Jilin Trust and Ping An Trust both operate in the shadows of the Chinese financial system, providing loans to borrowers'}\n",
      "aspectSet : article\n",
      "modelVersion : 1\n",
      "editorial : {'byline': 'By Simon Rabinovitch in Shanghai'}\n",
      "lifecycle : {'initialPublishDateTime': '2014-01-01T02:35:08Z', 'lastPublishDateTime': '2014-01-01T02:35:08Z'}\n",
      "title : {'title': 'China trusts bring contrasting fortunes'}\n",
      "_id : 6cfa00e8-6dd0-11e3-9d9b-00144feabdc0\n",
      "location : {'uri': 'http://www.ft.com/cms/s/0/6cfa00e8-6dd0-11e3-9d9b-00144feabdc0.html'}\n",
      "apiUrl : http://api.ft.com/content/items/v1/6cfa00e8-6dd0-11e3-9d9b-00144feabdc0\n",
      "q_info : {'executive': [{'q': {'api_key': 'warj9jjwsuayfqajgggf7ac8', 'offset': 0, 'begin_date': '2014-01-01', 'term': 'executive', 'end_date': '2014-02-28'}, 'term_category': 1}]}\n",
      "\n",
      "summary : {'excerpt': 'Digital music streaming accounted for about 10 per cent of UK recorded music revenues last year, with more than £100m'}\n",
      "aspectSet : article\n",
      "modelVersion : 1\n",
      "editorial : {'byline': 'By Daniel Thomas'}\n",
      "lifecycle : {'initialPublishDateTime': '2014-01-01T03:33:27Z', 'lastPublishDateTime': '2014-01-01T03:33:27Z'}\n",
      "title : {'title': 'Streaming boosts UK music industry'}\n",
      "_id : 75dd03d6-721c-11e3-9c5c-00144feabdc0\n",
      "location : {'uri': 'http://www.ft.com/cms/s/0/75dd03d6-721c-11e3-9c5c-00144feabdc0.html'}\n",
      "apiUrl : http://api.ft.com/content/items/v1/75dd03d6-721c-11e3-9c5c-00144feabdc0\n",
      "q_info : {'executive': [{'q': {'api_key': 'warj9jjwsuayfqajgggf7ac8', 'offset': 0, 'begin_date': '2014-01-01', 'term': 'executive', 'end_date': '2014-02-28'}, 'term_category': 1}]}\n",
      "\n",
      "summary : {'excerpt': 'Nespresso is aiming to boost the international sales of its coffee capsules, and devote more resources to “long”'}\n",
      "aspectSet : article\n",
      "modelVersion : 1\n",
      "editorial : {'byline': 'By James Shotter in Lausanne'}\n",
      "lifecycle : {'initialPublishDateTime': '2014-01-01T11:13:57Z', 'lastPublishDateTime': '2014-01-01T11:13:57Z'}\n",
      "title : {'title': 'Nespresso brews plans to see off rivals'}\n",
      "_id : b1f1c132-67da-11e3-8ada-00144feabdc0\n",
      "location : {'uri': 'http://www.ft.com/cms/s/0/b1f1c132-67da-11e3-8ada-00144feabdc0.html'}\n",
      "apiUrl : http://api.ft.com/content/items/v1/b1f1c132-67da-11e3-8ada-00144feabdc0\n",
      "q_info : {'executive': [{'q': {'api_key': 'warj9jjwsuayfqajgggf7ac8', 'offset': 0, 'begin_date': '2014-01-01', 'term': 'executive', 'end_date': '2014-02-28'}, 'term_category': 1}]}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for c in db.articles.find()[:3]:\n",
    "    for k in c:\n",
    "        print(k, ':', c[k])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}