New Yorker Fiction Scrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### New Yorker Fiction Scrape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import re\n",
    "import lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Scrape the fiction links.\"\"\"\n",
    "metadata = []\n",
    "base_url = 'https://www.newyorker.com/magazine/fiction/page/{}'\n",
    "indices = range(150)\n",
    "for i in indices:\n",
    "    try:\n",
    "        contents = requests.get(base_url.format(i)).content\n",
    "        tree = html.fromstring(contents)\n",
    "        titles = tree.xpath('//h4[@class=\"River__hed___re6RP\"]/text()')\n",
    "        titles = [re.sub(r'[^a-zA-Z ]+', '', t) for t in titles]\n",
    "        authors = tree.xpath('//a[@rel=\"author\"]/text()')\n",
    "        links = [l for l in tree.xpath('//div[@class=\"River__riverItemBody___347sz\"]//a[@class=\"Link__link___3dWao  \"]/@href') if 'magazine' in l]\n",
    "        assert len(titles) == len(authors)\n",
    "        assert len(titles) == len(links)\n",
    "        metadata += list(zip(titles, authors, links))\n",
    "    except Exception as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now on 0\n",
      "Now on 1\n",
      "Now on 2\n",
      "Now on 3\n",
      "Now on 4\n",
      "Now on 5\n",
      "Now on 6\n",
      "Error on:  Playing Metal Gear Solid V The Phantom Pain Jamil Jan Kochai /magazine/2020/01/06/playing-metal-gear-solid-v-the-phantom-pain\n",
      "Now on 7\n",
      "Now on 8\n",
      "Now on 9\n",
      "Now on 10\n",
      "Now on 11\n",
      "Now on 12\n",
      "Now on 13\n",
      "Now on 14\n",
      "Now on 15\n",
      "Now on 16\n",
      "Error on:  Playing Metal Gear Solid V The Phantom Pain Jamil Jan Kochai /magazine/2020/01/06/playing-metal-gear-solid-v-the-phantom-pain\n",
      "Now on 17\n",
      "Now on 18\n",
      "Now on 19\n",
      "Now on 20\n",
      "Error on:  The Curfew Roddy Doyle /magazine/2019/12/02/the-curfew\n",
      "Now on 21\n",
      "Now on 22\n",
      "Error on:  The Trip Weike Wang /magazine/2019/11/18/the-trip\n",
      "Now on 23\n",
      "Now on 24\n",
      "Now on 25\n",
      "Now on 26\n",
      "Now on 27\n",
      "Now on 28\n",
      "Now on 29\n",
      "Error on:  The Fellow Joy Williams /magazine/2019/09/30/the-fellow\n",
      "Now on 30\n",
      "Now on 31\n",
      "Now on 32\n",
      "Now on 33\n",
      "Error on:  To Do Kate Walbert /magazine/2019/09/02/to-do\n",
      "Now on 34\n",
      "Now on 35\n",
      "Error on:  Elliott Spencer George Saunders /magazine/2019/08/19/elliott-spencer\n",
      "Now on 36\n",
      "Now on 37\n",
      "Now on 38\n",
      "Now on 39\n",
      "Now on 40\n",
      "Error on:  Son of Friedman Emma Cline /magazine/2019/07/01/son-of-friedman\n",
      "Now on 41\n",
      "Now on 42\n",
      "Now on 43\n",
      "Now on 44\n",
      "Error on:  Conduction Ta-Nehisi Coates /magazine/2019/06/10/conduction\n",
      "Now on 45\n",
      "Now on 46\n",
      "Now on 47\n",
      "Now on 48\n",
      "Now on 49\n",
      "Error on:  The Escape John L’Heureux /magazine/2019/05/06/the-escape\n",
      "Now on 50\n",
      "Now on 51\n",
      "Now on 52\n",
      "Now on 53\n",
      "Now on 54\n",
      "Now on 55\n",
      "Now on 56\n",
      "Now on 57\n",
      "Now on 58\n",
      "Now on 59\n",
      "Now on 60\n",
      "Now on 61\n",
      "Now on 62\n",
      "Now on 63\n",
      "Now on 64\n",
      "Now on 65\n",
      "Now on 66\n",
      "Now on 67\n",
      "Now on 68\n",
      "Now on 69\n",
      "Now on 70\n",
      "Now on 71\n",
      "Now on 72\n",
      "Now on 73\n",
      "Error on:  Fiction by Bryan Washington Waugh Bryan Washington /magazine/2018/10/29/waugh\n",
      "Now on 74\n",
      "Now on 75\n",
      "Error on:  Fiction by Kevin Barry The Coast of Leitrim Kevin Barry /magazine/2018/10/15/the-coast-of-leitrim\n",
      "Now on 76\n",
      "Now on 77\n",
      "Now on 78\n",
      "Now on 79\n",
      "Now on 80\n",
      "Now on 81\n",
      "Now on 82\n",
      "Now on 83\n",
      "Now on 84\n",
      "Now on 85\n",
      "Now on 86\n",
      "Now on 87\n",
      "Now on 88\n",
      "Now on 89\n",
      "Now on 90\n",
      "Now on 91\n",
      "Now on 92\n",
      "Now on 93\n",
      "Now on 94\n",
      "Now on 95\n",
      "Now on 96\n",
      "Now on 97\n",
      "Now on 98\n",
      "Now on 99\n",
      "Now on 100\n",
      "Now on 101\n",
      "Now on 102\n",
      "Now on 103\n",
      "Now on 104\n",
      "Now on 105\n",
      "Now on 106\n",
      "Now on 107\n",
      "Now on 108\n",
      "Now on 109\n",
      "Now on 110\n",
      "Now on 111\n",
      "Now on 112\n",
      "Now on 113\n",
      "Now on 114\n",
      "Now on 115\n",
      "Error on:  Cat Person Kristen Roupenian /magazine/2017/12/11/cat-person\n",
      "Now on 116\n",
      "Now on 117\n",
      "Now on 118\n",
      "Now on 119\n",
      "Now on 120\n",
      "Now on 121\n",
      "Now on 122\n",
      "Now on 123\n",
      "Now on 124\n",
      "Now on 125\n",
      "Now on 126\n",
      "Now on 127\n",
      "Now on 128\n",
      "Now on 129\n",
      "Now on 130\n",
      "Now on 131\n",
      "Now on 132\n",
      "Now on 133\n",
      "Now on 134\n",
      "Now on 135\n",
      "Now on 136\n",
      "Now on 137\n",
      "Now on 138\n",
      "Now on 139\n",
      "Now on 140\n",
      "Now on 141\n",
      "Now on 142\n",
      "Now on 143\n",
      "Now on 144\n",
      "Now on 145\n",
      "Now on 146\n",
      "Now on 147\n",
      "Now on 148\n",
      "Now on 149\n",
      "Now on 150\n",
      "Now on 151\n",
      "Error on:  Herman Melville Volume I Victor Lodato /magazine/2017/03/27/herman-melville-volume-i\n",
      "Now on 152\n",
      "Now on 153\n",
      "Error on:  Solstice Anne Enright /magazine/2017/03/13/solstice\n",
      "Now on 154\n",
      "Now on 155\n",
      "Now on 156\n",
      "Now on 157\n",
      "Now on 158\n",
      "Now on 159\n",
      "Now on 160\n",
      "Now on 161\n",
      "Now on 162\n",
      "Now on 163\n",
      "Now on 164\n",
      "Now on 165\n",
      "Now on 166\n",
      "Now on 167\n",
      "Now on 168\n",
      "Error on:  Of Windows and Doors Mohsin Hamid /magazine/2016/11/14/of-windows-and-doors\n",
      "Now on 169\n",
      "Now on 170\n",
      "Now on 171\n",
      "Now on 172\n",
      "Now on 173\n",
      "Now on 174\n",
      "Now on 175\n",
      "Now on 176\n",
      "Now on 177\n",
      "Now on 178\n",
      "Now on 179\n",
      "Now on 180\n",
      "Now on 181\n",
      "Now on 182\n",
      "Now on 183\n",
      "Now on 184\n",
      "Now on 185\n",
      "Now on 186\n",
      "Now on 187\n",
      "Error on:  Bog Girl Karen Russell /magazine/2016/06/20/bog-girl-by-karen-russell\n",
      "Now on 188\n",
      "Now on 189\n",
      "Now on 190\n",
      "Now on 191\n",
      "Error on:  Seven People Dancing Langston Hughes /magazine/2016/06/06/seven-people-dancing-by-langston-hughes\n",
      "Now on 192\n",
      "Now on 193\n",
      "Now on 194\n",
      "Now on 195\n",
      "Now on 196\n",
      "Now on 197\n",
      "Now on 198\n",
      "Now on 199\n",
      "Now on 200\n",
      "Now on 201\n",
      "Error on:  My Purple Scented Novel Ian McEwan /magazine/2016/03/28/my-purple-scented-novel-fiction-by-ian-mcewan\n",
      "Now on 202\n",
      "Now on 203\n",
      "Now on 204\n",
      "Now on 205\n",
      "Now on 206\n",
      "Now on 207\n",
      "Now on 208\n",
      "Now on 209\n",
      "Error on:  Aspic Tatyana Tolstaya /magazine/2016/01/25/aspic-fiction-tatyana-tolstaya\n",
      "Now on 210\n",
      "Error on:  The Story of a Painter Ludmilla Petrushevskaya /magazine/2016/01/18/the-story-of-a-painter\n",
      "Now on 211\n",
      "Now on 212\n",
      "Now on 213\n",
      "Now on 214\n",
      "Now on 215\n",
      "Now on 216\n",
      "Now on 217\n",
      "Now on 218\n",
      "Now on 219\n",
      "Error on:  Honey Bunny Julianne Pachico /magazine/2015/11/09/honey-bunny\n",
      "Now on 220\n",
      "Now on 221\n",
      "Now on 222\n",
      "Now on 223\n",
      "Now on 224\n",
      "Now on 225\n",
      "Now on 226\n",
      "Error on:  My Curls Have Blown All the Way to China Amos Oz /magazine/2015/09/21/my-curls-have-blown-all-the-way-to-china\n",
      "Now on 227\n",
      "Now on 228\n",
      "Now on 229\n",
      "Now on 230\n",
      "Now on 231\n",
      "Now on 232\n",
      "Now on 233\n",
      "Now on 234\n",
      "Now on 235\n",
      "Now on 236\n",
      "Now on 237\n",
      "Now on 238\n",
      "Now on 239\n",
      "Now on 240\n",
      "Now on 241\n",
      "Now on 242\n",
      "Now on 243\n",
      "Now on 244\n",
      "Now on 245\n",
      "Error on:  So Youre Just What Gone Justin Taylor /magazine/2015/05/18/so-youre-just-what-gone\n",
      "Now on 246\n",
      "Now on 247\n",
      "Now on 248\n",
      "Now on 249\n",
      "Now on 250\n",
      "Now on 251\n",
      "Now on 252\n",
      "Now on 253\n",
      "Now on 254\n",
      "Now on 255\n",
      "Now on 256\n",
      "Now on 257\n",
      "Now on 258\n",
      "Now on 259\n",
      "Error on:  Sweetness Toni Morrison /magazine/2015/02/09/sweetness-2\n",
      "Now on 260\n",
      "Now on 261\n",
      "Now on 262\n",
      "Error on:  Breadman J. Robert Lennon /magazine/2015/01/19/breadman\n",
      "Now on 263\n",
      "Now on 264\n",
      "Error on:  The Ways Colin Barrett /magazine/2015/01/05/ways-2\n",
      "Now on 265\n",
      "Now on 266\n",
      "Now on 267\n",
      "Now on 268\n",
      "Now on 269\n",
      "Now on 270\n",
      "Now on 271\n",
      "Now on 272\n",
      "Now on 273\n",
      "Error on:  Alan Bean Plus Four Tom Hanks /magazine/2014/10/27/alan-bean-plus-four\n",
      "Now on 274\n",
      "Now on 275\n",
      "Now on 276\n",
      "Now on 277\n",
      "Error on:  Rosendale Paul La Farge /magazine/2014/09/29/rosendale\n",
      "Now on 278\n",
      "Now on 279\n",
      "Now on 280\n",
      "Now on 281\n",
      "Now on 282\n",
      "Now on 283\n",
      "Now on 284\n",
      "Now on 285\n",
      "Now on 286\n",
      "Now on 287\n",
      "Now on 288\n",
      "Now on 289\n",
      "Now on 290\n",
      "Now on 291\n",
      "Error on:  Madame Lazarus Maile Meloy /magazine/2014/06/23/madame-lazarus\n",
      "Now on 292\n",
      "Error on:  Yesterday Haruki Murakami /magazine/2014/06/09/yesterday-3\n",
      "Now on 293\n",
      "Now on 294\n",
      "Now on 295\n",
      "Now on 296\n",
      "Error on:  Ba Baboon Thomas Pierce /magazine/2014/06/02/ba-baboon\n",
      "Now on 297\n",
      "Now on 298\n",
      "Now on 299\n",
      "Now on 300\n",
      "Now on 301\n",
      "Now on 302\n",
      "Now on 303\n",
      "Now on 304\n",
      "Now on 305\n",
      "Now on 306\n",
      "Now on 307\n",
      "Now on 308\n",
      "Now on 309\n",
      "Now on 310\n",
      "Now on 311\n",
      "Now on 312\n",
      "Now on 313\n",
      "Now on 314\n",
      "Now on 315\n",
      "Now on 316\n",
      "Now on 317\n",
      "Now on 318\n",
      "Now on 319\n",
      "Now on 320\n",
      "Now on 321\n",
      "Now on 322\n",
      "Now on 323\n",
      "Now on 324\n",
      "Now on 325\n",
      "Now on 326\n",
      "Now on 327\n",
      "Error on:  The Bear Came Over the Mountain Alice Munro /magazine/2013/10/21/the-bear-came-over-the-mountain-2\n",
      "Now on 328\n",
      "Now on 329\n",
      "Now on 330\n",
      "Now on 331\n",
      "Now on 332\n",
      "Now on 333\n",
      "Now on 334\n",
      "Now on 335\n",
      "Now on 336\n",
      "Now on 337\n",
      "Now on 338\n",
      "Now on 339\n",
      "Error on:  From a Farther Room David Gilbert /magazine/2013/07/22/from-a-farther-room\n",
      "Now on 340\n",
      "Now on 341\n",
      "Now on 342\n",
      "Error on:  Stars Thomas McGuane /magazine/2013/06/24/stars-3\n",
      "Now on 343\n",
      "Now on 344\n",
      "Now on 345\n",
      "Now on 346\n",
      "Now on 347\n",
      "Error on:  Slide To Unlock Ed Park /magazine/2013/06/10/slide-to-unlock\n",
      "Now on 348\n",
      "Now on 349\n",
      "Now on 350\n",
      "Now on 351\n",
      "Now on 352\n",
      "Now on 353\n",
      "Now on 354\n",
      "Now on 355\n",
      "Now on 356\n",
      "Now on 357\n",
      "Now on 358\n",
      "Now on 359\n",
      "Now on 360\n",
      "Now on 361\n",
      "Now on 362\n",
      "Now on 363\n",
      "Now on 364\n",
      "Now on 365\n",
      "Now on 366\n",
      "Now on 367\n",
      "Now on 368\n",
      "Now on 369\n",
      "Now on 370\n",
      "Now on 371\n",
      "Now on 372\n",
      "Now on 373\n",
      "Now on 374\n",
      "Now on 375\n",
      "Error on:  Member  Guest David Gilbert /magazine/2012/11/12/member-guest\n",
      "Now on 376\n",
      "Now on 377\n",
      "Now on 378\n",
      "Now on 379\n",
      "Now on 380\n",
      "Now on 381\n",
      "Now on 382\n",
      "Now on 383\n",
      "Now on 384\n",
      "Now on 385\n",
      "Now on 386\n",
      "Now on 387\n",
      "Now on 388\n",
      "Now on 389\n",
      "Now on 390\n",
      "Now on 391\n",
      "Now on 392\n",
      "Now on 393\n",
      "Now on 394\n",
      "Error on:  The Republic of Empathy Sam Lipsyte /magazine/2012/06/04/the-republic-of-empathy\n",
      "Now on 395\n",
      "Now on 396\n",
      "Now on 397\n",
      "Now on 398\n",
      "Error on:  Referential Lorrie Moore /magazine/2012/05/28/referential\n",
      "Now on 399\n",
      "Now on 400\n",
      "Now on 401\n",
      "Now on 402\n",
      "Now on 403\n",
      "Now on 404\n",
      "Error on:  Transatlantic Colum McCann /magazine/2012/04/16/transatlantic-3\n",
      "Now on 405\n",
      "Now on 406\n",
      "Now on 407\n",
      "Error on:  Chapter Two Antonya Nelson /magazine/2012/03/26/chapter-two\n",
      "Now on 408\n",
      "Now on 409\n",
      "Now on 410\n",
      "Error on:  Haven Alice Munro /magazine/2012/03/05/haven-alice-munro\n",
      "Now on 411\n",
      "Now on 412\n",
      "Now on 413\n",
      "Now on 414\n",
      "Now on 415\n",
      "Now on 416\n",
      "Now on 417\n",
      "Now on 418\n",
      "Now on 419\n",
      "Now on 420\n",
      "Now on 421\n",
      "Now on 422\n",
      "Now on 423\n",
      "Now on 424\n",
      "Now on 425\n",
      "Now on 426\n",
      "Now on 427\n",
      "Now on 428\n",
      "Now on 429\n",
      "Now on 430\n",
      "Now on 431\n",
      "Now on 432\n",
      "Now on 433\n",
      "Now on 434\n",
      "Now on 435\n",
      "Now on 436\n",
      "Now on 437\n",
      "Now on 438\n",
      "Now on 439\n",
      "Now on 440\n",
      "Now on 441\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error on:  Asleep in the Lord Jeffrey Eugenides /magazine/2011/06/13/asleep-in-the-lord\n",
      "Now on 442\n",
      "Now on 443\n",
      "Now on 444\n",
      "Now on 445\n",
      "Now on 446\n",
      "Now on 447\n",
      "Now on 448\n",
      "Now on 449\n",
      "Now on 450\n",
      "Now on 451\n",
      "Now on 452\n",
      "Now on 453\n",
      "Now on 454\n",
      "Now on 455\n",
      "Now on 456\n",
      "Now on 457\n",
      "Now on 458\n",
      "Now on 459\n",
      "Now on 460\n",
      "Now on 461\n",
      "Now on 462\n",
      "Now on 463\n",
      "Now on 464\n",
      "Now on 465\n",
      "Now on 466\n",
      "Now on 467\n",
      "Now on 468\n",
      "Now on 469\n",
      "Now on 470\n",
      "Now on 471\n",
      "Error on:  The Trojan Prince Tessa Hadley /magazine/2010/11/15/the-trojan-prince\n",
      "Now on 472\n",
      "Now on 473\n",
      "Now on 474\n",
      "Now on 475\n",
      "Error on:  To the Measures Fall Richard Powers /magazine/2010/10/18/to-the-measures-fall\n",
      "Now on 476\n",
      "Now on 477\n",
      "Now on 478\n",
      "Now on 479\n",
      "Now on 480\n",
      "Error on:  Ash Roddy Doyle /magazine/2010/05/24/ash\n",
      "Now on 481\n",
      "Now on 482\n",
      "Error on:  Uncle Rock Dagoberto Gilb /magazine/2010/05/10/uncle-rock\n",
      "Now on 483\n",
      "Now on 484\n",
      "Error on:  Edgemont Drive E. L. Doctorow /magazine/2010/04/26/edgemont-drive\n",
      "Now on 485\n",
      "Now on 486\n",
      "Now on 487\n",
      "Now on 488\n",
      "Now on 489\n",
      "Now on 490\n",
      "Now on 491\n",
      "Now on 492\n",
      "Error on:  Appetite Saïd Sayrafiezadeh /magazine/2010/03/01/appetite\n",
      "Now on 493\n",
      "Now on 494\n",
      "Now on 495\n",
      "Now on 496\n",
      "Now on 497\n",
      "Now on 498\n",
      "Now on 499\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Scrape the fiction body text from the links.\"\"\"\n",
    "base_url = 'https://www.newyorker.com/{}'\n",
    "fiction = []\n",
    "\n",
    "def body_text(raw_text):\n",
    "    text = [t for t in raw_text if len(t) > 36]\n",
    "    end = [i for (i, t) in enumerate(text) if '♦' in t]\n",
    "    if len(end) < 1:\n",
    "        return None\n",
    "    end = end[0]\n",
    "    text = text[:end]\n",
    "    return text\n",
    "\n",
    "for i, (t, a, l) in enumerate(metadata[:500]):\n",
    "    print(\"Now on {}\".format(i))\n",
    "    try:\n",
    "        contents = requests.get(base_url.format(l)).content\n",
    "        tree = html.fromstring(contents)\n",
    "        raw_text = tree.xpath('//p/text()')\n",
    "        body = body_text(raw_text)\n",
    "        if body is None: assert False\n",
    "        fiction += [(t, a, l, body)]\n",
    "    except Exception as e:\n",
    "        print(\"Error on: \", t, a, l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('newyorker_fiction_raw.p', 'wb') as f:\n",
    "    pickle.dump(fiction,f )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one 7023\n",
      "('With the Beatles', 'Haruki Murakami', ('There’s', ' one ', 'girl—a woman who used to be a girl, I mean—whom I remember well. I don’t know her name, though. And, naturally, I don’t know where she is now or what she’s doing. What I do know about her is that she went to the same high school as I did, and was in the same year (since the badge on her shirt was the same color as mine), and that she really liked the Beatles.'))\n",
      "two 2700\n",
      "('With the Beatles', 'Haruki Murakami', ('That was the only time I saw that girl. In the', ' two ', 'years between then and my graduation, we never once crossed paths again. Which is pretty strange if you think about it. The high school I attended was a fairly large public school at the top of a hill in Kobe, with about six hundred and fifty students in each grade. (We were the so-called baby-boomer generation, so there were a lot of us.) Not everyone knew one another. In fact, I didn’t know the names or recognize the vast majority of the kids in the school. But, still, since I went to school almost every day, and often used that hallway, it struck me as almost outrageous that I never once saw that beautiful girl again. I looked for her every time I used that hallway.'))\n",
      "three 1192\n",
      "('With the Beatles', 'Haruki Murakami', ('I never saw him again. Chance had brought us together a second time. With nearly twenty years between encounters, in cities', ' three ', 'hundred miles apart, we’d sat, a table between us, sipping coffee and talking over a few things. But these weren’t subjects you just chatted about over coffee. There was something more significant in our talk, something that seemed meaningful to us, in the act of living out our lives. Still, it was merely a hint, delivered by chance. There was nothing to link us together in a more systematic or organic way. (Question: '))\n",
      "four 524\n",
      "('With the Beatles', 'Haruki Murakami', ('This was in 1964, at the height of Beatlemania. It was early autumn. The new school semester had begun and things were starting to fall into a routine again. She was hurrying down the long, dim hallway of the old school building, her skirt fluttering. I was the only other person there. She was clutching an LP to her chest as if it were something precious. The LP “With the Beatles.” The one with the striking black-and-white photograph of the', ' four ', 'Beatles in half shadow. For some reason, I’m not sure why, I have a clear memory that it was the original, British version of the album, not the American or the Japanese version.'))\n",
      "five 489\n",
      "('With the Beatles', 'Haruki Murakami', (' wildly popular. By April of 1964, they’d captured the top', ' five ', 'spots on the American singles charts. Pop music had never seen anything like it. These were the'))\n",
      "six 387\n",
      "('With the Beatles', 'Haruki Murakami', ('That was the only time I saw that girl. In the two years between then and my graduation, we never once crossed paths again. Which is pretty strange if you think about it. The high school I attended was a fairly large public school at the top of a hill in Kobe, with about', ' six ', 'hundred and fifty students in each grade. (We were the so-called baby-boomer generation, so there were a lot of us.) Not everyone knew one another. In fact, I didn’t know the names or recognize the vast majority of the kids in the school. But, still, since I went to school almost every day, and often used that hallway, it struck me as almost outrageous that I never once saw that beautiful girl again. I looked for her every time I used that hallway.'))\n",
      "seven 179\n",
      "('The Flier', 'Joseph O’Neill', ('Pam and Becky arrived on the dot, at', ' seven.', ' My illness had made me very small and very light, and they embraced me gently. “He looks so young,” Becky said to Viki. “Where’s Molly?”'))\n",
      "eight 225\n",
      "('With the Beatles', 'Haruki Murakami', ('When I was in my mid-thirties, well past childhood and adolescence, my first impression of the album was that it wasn’t that great, or at least not the kind of music to take your breath away. Of the fourteen tracks on the album, six were covers of other artists’ works. The covers of the Marvelettes’ “Please Mr. Postman” and Chuck Berry’s “Roll Over Beethoven” were well done, and impress me even when I listen to them now, but, still, they were cover versions. And of the', ' eight ', 'original songs, apart from Paul’s “All My Loving,” none were amazing. There were no hit singles, and to my ears the Beatles’ first album, “Please Please Me,” recorded basically in one take, was far more vibrant and compelling. Even so, likely thanks to Beatles fans’ unquenchable desire for new songs, this second album débuted in the No.\\xa01 spot in the U.K., a position it held for twenty-one weeks. (In the U.S., the title of the album was changed to “Meet the Beatles,” and included some different tracks, though the cover design stayed almost the same.)'))\n",
      "nine 146\n",
      "('Things We Worried About When I Was Ten', 'David Rabe', (' again. “What the hell now?” my dad griped. It went on and on, and Dad couldn’t figure out what it was, and Mom couldn’t, either. It started at about', ' nine ', 'and went on till ten or later, and Dad was on his way to complain to the landlord, whose house was next door, when he decided instead to talk to Agnes Rath, who lived in the apartment under us. It turned out that Agnes was scared sick. When Dad knocked, she turned on her porch light and peeked out between her curtains, and, seeing that it was him, she opened her door and told him that Mr. Stink had been peeping in her window. She’d seen him and, not knowing what to do, had turned off all her lights and crawled into the kitchen. Lying on the floor, she’d banged on the pipes under her sink as a signal. So that was the clanking. Agnes Rath’s signal. Well, a few nights later, a group of men ran through our yard and my dad ran with them, and then, not too long after that, fire leaped up on the hill around the spot where Mr. Stink had his shack, and nobody ever saw him again.'))\n",
      "ten 478\n",
      "('With the Beatles', 'Haruki Murakami', ('My heart started to pound, I gasped for breath, and it was as if all sound had ceased, as if I’d sunk to the bottom of a pool. All I could hear was a bell ringing faintly, deep in my ears. As if someone were desperately trying to send me a vital message. All this took only', ' ten ', 'or fifteen seconds. It was over before I knew it, and the critical message contained there, like the core of all dreams, disappeared.'))\n",
      "eleven 71\n",
      "('With the Beatles', 'Haruki Murakami', ('“That’s right,” I said and introduced myself. “I was supposed to come here at', ' eleven.', '”'))\n",
      "twelve 140\n",
      "('With the Beatles', 'Haruki Murakami', ('Whether you liked the writing or not, one thing was clear: this wasn’t the right story to read on a bright, clear Sunday. I closed the book and glanced up at the clock on the wall. It was just past', ' twelve.', ''))\n",
      "zero 36\n",
      "('The Little King', 'Salman Rushdie', ('Yes, unfortunately, some of them got investigated by their state medical boards, but they just handled it! They paid the fines and carried on. Yes, unfortunately, in the worst cases there was disciplinary action when, unfortunately, some of the tough doctors went too far! When, unfortunately, they allegedly handed out pre-signed prescriptions to patients and some of said patients died of drug overdoses from the drugs so prescribed! When, unfortunately, they allegedly prescribed InSmile™ to persons with', ' zero ', 'cancer pain! When, unfortunately, they allegedly defrauded Medicare of multiple millions of dollars! A pain-management specialist from Rhode Island who was also a '))\n",
      "nothing 1495\n",
      "('With the Beatles', 'Haruki Murakami', ('I’ve heard it said that the happiest time in our lives is the period when pop songs really mean something to us, really get to us. It may be true. Or maybe not. Pop songs may, after all, be', ' nothing ', 'but pop songs. And perhaps our lives are merely decorative, expendable items, a burst of fleeting color and'))\n",
      "fifteen 183\n",
      "('With the Beatles', 'Haruki Murakami', ('My heart started to pound, I gasped for breath, and it was as if all sound had ceased, as if I’d sunk to the bottom of a pool. All I could hear was a bell ringing faintly, deep in my ears. As if someone were desperately trying to send me a vital message. All this took only ten or', ' fifteen ', 'seconds. It was over before I knew it, and the critical message contained there, like the core of all dreams, disappeared.'))\n",
      "thirty 170\n",
      "('With the Beatles', 'Haruki Murakami', ('“You’ve come all this way, so how about waiting another', ' thirty ', 'minutes?” her brother asked. “How about you wait another'))\n",
      "forty-five 38\n",
      "('Arizona', 'John Edgar Wideman', ('Distance from Phoenix to Flagstaff, Arizona approximately one hundred and', ' forty-five ', 'miles. A drive north of about two hours, eighteen minutes on I-17. If you are interested, time-lapse videos posted on the Internet can get you there faster, in anywhere from 1:42 to 47:10. The two lawyers, one in charge, one to drive, who met my son’s plane at Sky Harbor airport in order to escort him to a jail in Flagstaff, where he’d be locked up until tried for murder, may or may not have been in a hurry. Being in a hurry doesn’t necessarily get you any quicker to where you wish to go. Nor does the wish to never arrive at a particular destination necessarily retard arrival. I wasn’t in the car hauling my son to jail that day in Arizona, thus can’t say who was in a hurry and who was not. My son a fugitive for twelve days before he called his uncle and asked for help. Before his uncle called us and we engaged lawyers. How do my son’s twelve days of running compare to my old buddy’s seventeen years of flight and hiding. I should know better than to ask such a silly question, Mr. Jackson. As if aging and loss and cancer and fear and mourning and despair are not species of time. As if instead, time consisted of a certain quantity of repeating, unchanging, definable units, like inches, miles, pounds. As if the length and weight of a boy’s time on the road were measurable, a fifteen-year-old who hasn’t maybe had sex yet or attended a funeral or slept alone away from home a single night with no family, no adult, no companions keeping him company, as if such a boy’s terror after killing, for no reason, he comprehends his roommate in an Arizona motel on a pleasure trip supervised by an expensive, élite boys’ camp in Vermont, as if anyone not that boy might grasp how time passed for him on the run or passes now in prison where bars and cages do not stop the running, but torture and bend time so time collapses, empties, or swells like a corpse decomposing, or towers like some suppurating beast many stories tall with bloody talons pawing the air. Stink, moans, a dreadful roaring to halt my son in his tracks, keep him fleeing.'))\n",
      "forty 107\n",
      "('Found Wanting', 'Douglas Stuart', (' It was only', ' forty ', 'miles away, and I had never been. I was scared, but underneath my fear I was buzzing. The traffic moved quickly at this time of night. The rain-dappled windscreen was a constellation of orange and red. The car settled into fifth gear, and the Solicitor put his hand on my leg. I liked it. I liked the possessiveness of it, how it weighted me down. I hadn’t realized how tired I was until his hand grasped me. I felt as if I had been found somehow. I watched the slower cars fall away.'))\n",
      "midnight 69\n",
      "('Three Women of Chucks Donuts', 'Anthony Veasna So', ('By this point, Sothy, Tevy, and Kayley have grown accustomed to the man’s presence, aware that on any given night he may appear sometime between', ' midnight ', 'and four. The daughters whisper about him, half hoping that where he sits is out of earshot, half hoping he’ll overhear them. Kayley speculates about his motives: if he’s a police officer on a stakeout, say, or a criminal on the run. She deliberates over whether he’s a good man or a bad one. Tevy, on the other hand, theorizes about the man’s purpose—if, for example, he feels detached from the world and can center himself only here, in Chuck’s Donuts, around other Khmer people. Both sisters wonder about his life: the kind of women he attracts and has dated; the women he has spurned; whether he has siblings, or kids; whether he looks more like his mother or father.'))\n",
      "noon 52\n",
      "('The Children', 'Andrea Lee', ('It is almost', ' noon, ', 'and the Fleur des Îles is filling up with rich Malagasy and Indian kids from Lycée Sacré Cœur, devouring '))\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Grab body text with numbers.\"\"\"\n",
    "times = {\n",
    "    'hours' : {k: [] for k in ['one', 'two', 'three', 'four', 'five', 'six', 'seven',\n",
    "                         'eight', 'nine', 'ten', 'eleven', 'twelve']},\n",
    "    'minutes' : {k: [] for k in ['zero', 'nothing', 'fifteen', 'thirty', 'forty-five', 'forty']},\n",
    "    'special' : {k: [] for k in ['midnight', 'noon']}\n",
    "}\n",
    "\n",
    "\n",
    "def extract_times(times, body, title, author):\n",
    "    for t in body:\n",
    "        match_templates = [' {} ', ' {}, ', ' {}.']\n",
    "        for time_type in times:\n",
    "            for k in times[time_type]:\n",
    "                for match_template in match_templates:\n",
    "                    if match_template.format(k) in t:\n",
    "                        # Clean up to make everything a sentence.\n",
    "                        t = t.replace(' . . . ', '...')\n",
    "\n",
    "                        splits = t.split(match_template.format(k))\n",
    "                        for i in range(len(splits) - 1):\n",
    "                            times[time_type][k].append(\n",
    "                                (title, author, \n",
    "                                (splits[i], match_template.format(k), splits[i+1])\n",
    "                                )\n",
    "                            )\n",
    "                            \n",
    "                            \n",
    "    return times\n",
    "\n",
    "for i, (t, a, l, body) in enumerate(fiction):\n",
    "    times = extract_times(times, body, t, a)\n",
    "\n",
    "for time_type in times:\n",
    "    for k in times[time_type]:\n",
    "        print(k, len(times[time_type][k]))\n",
    "        if len(times[time_type][k]) > 0:\n",
    "            print(times[time_type][k][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"POS tag the times with templates for completion.\"\"\"\n",
    "fragment_templates = [\n",
    "#     ('article', 'noun'), # (the, a, an); (noun)\n",
    "#     ('pronoun be', '')\n",
    "#         ('and', 'and'),\n",
    "#         ('or', 'or'),\n",
    "        ('pronoun_be', 'pronoun_be'),\n",
    "        ('pronoun_has', 'pronoun_has'),\n",
    "        ('pronoun_will', 'pronoun_will')\n",
    "]\n",
    "\n",
    "from collections import defaultdict\n",
    "fragments = {time_type : {k : defaultdict(list) for k in times[time_type]} for time_type in times}\n",
    "\n",
    "max_chars = 300\n",
    "min_chars = 150\n",
    "for time_type in times:\n",
    "    for k in times[time_type]:\n",
    "        for (title, author, (before, match, after)) in times[time_type][k]:\n",
    "            # Break on conjunctions.\n",
    "#             conjunctions = [' and ', ' or ']\n",
    "            conjunctions = []\n",
    "            pronoun_be = [' he was ', ' he is ', ' He was ', ' He is ', ' She was ', ' She is ', ' she is ', ' she was ', ' I am ', ' they are ', ' they were ']\n",
    "            pronoun_has = [' he has ', ' she has ', ' they had ', ' I have ']\n",
    "            pronoun_will = [' he will ', ' she will ', ' I will ', \" He will \", \" She will \"]\n",
    "            \n",
    "            for splitter in conjunctions + pronoun_be + pronoun_has + pronoun_will:\n",
    "                if time_type == 'hours':\n",
    "                    before_splits = before.split(\". \")\n",
    "                    after_splits = after.split(splitter)\n",
    "                    for i in range(1, len(after_splits)):\n",
    "                        frag_end = splitter.join(after_splits[:i]) + splitter\n",
    "                        if len(frag_end) > max_chars: continue\n",
    "                        # Go backwards over the sentence starts.\n",
    "                        for j in range(1, len(before_splits)):\n",
    "                            frag_start = \". \".join(before_splits[::-1][:j][::-1])\n",
    "                            if len(frag_start) + len(frag_end) > max_chars: continue\n",
    "                            if len(frag_start) + len(frag_end) < min_chars: continue\n",
    "                            \n",
    "                            if splitter in conjunction:\n",
    "                                lookup_key = splitter.strip()\n",
    "                            if splitter in pronoun_be:\n",
    "                                lookup_key = 'pronoun_be'\n",
    "                            if splitter in pronoun_has:\n",
    "                                lookup_key = 'pronoun_has'\n",
    "                            if splitter in pronoun_will:\n",
    "                                lookup_key = 'pronoun_will'\n",
    "                                \n",
    "                            fragments[time_type][k][lookup_key].append(\n",
    "                                ((frag_start, match, frag_end), title, author)\n",
    "                            )                   \n",
    "                elif time_type == 'minutes':\n",
    "                    # Whatever comes after the conjunction, and then next sentences.\n",
    "                    before_splits = before.split(splitter)\n",
    "                    after_splits = after.split(\". \")\n",
    "                    for j in range(1, len(before_splits)):\n",
    "                        frag_start = splitter.join(before_splits[::-1][:j][::-1])\n",
    "                        for i in range(1, len(after_splits)):\n",
    "                            frag_end = \". \".join(after_splits[:i])\n",
    "                            total_len = len(frag_start) + len(frag_end)\n",
    "                            if total_len > min_chars and total_len < max_chars:\n",
    "                                \n",
    "                                if splitter in conjunction:\n",
    "                                    lookup_key = splitter.strip()\n",
    "                                if splitter in pronoun_be:\n",
    "                                    lookup_key = 'pronoun_be'\n",
    "                                if splitter in pronoun_has:\n",
    "                                    lookup_key = 'pronoun_has'\n",
    "                                if splitter in pronoun_will:\n",
    "                                    lookup_key = 'pronoun_will'\n",
    "                                \n",
    "                                fragments[time_type][k][lookup_key].append(\n",
    "                                ((frag_start, match, frag_end), title, author)\n",
    "                            )\n",
    "                elif time_type == 'special':\n",
    "                    # Sentences for the special times.\n",
    "                    splitter = '. '\n",
    "                    before_splits = before.split(splitter)\n",
    "                    after_splits = after.split(\". \")\n",
    "                    for j in range(1, len(before_splits)):\n",
    "                        frag_start = splitter.join(before_splits[::-1][:j][::-1])\n",
    "                        for i in range(1, len(after_splits)):\n",
    "                            frag_end = \". \".join(after_splits[:i])\n",
    "                            total_len = len(frag_start) + len(frag_end)\n",
    "                            if total_len > min_chars and total_len < max_chars:                       \n",
    "                                fragments[time_type][k]['special'].append(\n",
    "                                ((frag_start, match, frag_end), title, author)\n",
    "                            )\n",
    "                    \n",
    "        \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine all of the overloaded times.\n",
    "can_combine = {\n",
    "                    'forties' : ['forty-five', 'forty'],\n",
    "                    'zeros' : ['zero', 'nothing']\n",
    "                }\n",
    "for t2_type in can_combine:\n",
    "    for t2 in can_combine[t2_type]:\n",
    "        other_types = [o for o in can_combine[t2_type] if o != t2]\n",
    "        for time_type in fragments:\n",
    "            for k in fragments[time_type]:\n",
    "                if k == t2:\n",
    "                    for frag_template in fragments[time_type][k]:\n",
    "                        for o in other_types:\n",
    "                            fragments[time_type][k][frag_template] += fragments[time_type][o][frag_template]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump to JSON\n",
    "import json\n",
    "with open('newyorker_fragments.json', 'w') as f:\n",
    "    data = {\n",
    "        'fragments' : fragments,\n",
    "        'fragment_templates': fragment_templates\n",
    "    }\n",
    "    json.dump(data, f)\n",
    "    \n",
    "with open('newyorker_fragments.json', 'r') as f:\n",
    "    data = json.load(f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('ten', 'forty-five')\n",
      "(('“This trip? Six countries.” And', ' ten ', 'cities. “Oh, definitely. And I’m reading all I can find on it, and everyone’s talking about nothing else. Well, I only met literary types, no politicos, no experts. But of course I have '), 'Oktober', 'Martin Amis')\n",
      "(('turned off the engine, there are hardly any other cars on the ferry, I wave to the ferryman and he waves back. I believe he’s been a ferryman here for', ' forty ', 'years. Summer is finally over'), 'Time for the Eyes to Adjust', 'Linn Ullmann')\n",
      "['midnight']\n",
      "(('She sleeps hard until', ' midnight, ', 'when a car in the driveway wakes her up. She’s not in bed but at her desk, half her toenails painted dark red, the computer printing a ninety-page document about the dangers of lead-based paint. Her pills can have that effect sometimes'), 'Shirley Temple Three', 'Thomas Pierce')\n",
      "None\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAJ+CAYAAACZ7IaeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAXEUlEQVR4nO3cf6zdd33f8dd7MYEq3XCSelZkWwsVViP+KCG9YkFUU5eIKUkrnD8oAlWLhSy5f2QTqJW6dJM2Vdof8E9TIk2RLMLqTAxI06JYKGqXmVTV/kjAQBp+pDSXiCi2kthAEmhRy9J+9sf9GC7GzNd535tzbB4P6eh8v5/v5/h87kfAU+d7z6XGGAGAV+qfLHoBAFzYhASAFiEBoEVIAGgREgBahASAli0JSVXdVFVfq6rVqrpjK94DgOVQm/13JFV1SZK/TvKOJMeTfC7Je8cYX93UNwJgKWzFJ5K3JlkdYzw1xvh+kk8k2bcF7wPAEti2Bf/mriTPrDs/nuRfnjmpqg4mOZgkl1122S9dc801W7AUADbDN77xjXzzm9+ss13bipBsyBjjUJJDSbKysjKOHTu2qKUAcA4rKys/8dpW3No6kWTPuvPdcwyAi9BWhORzSfZW1Ruq6tIk70lyZAveB4AlsOm3tsYYL1fVv0vyZ0kuSfLRMcZXNvt9AFgOW/I7kjHGg0ke3Ip/G4Dl4i/bAWgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAFiEBoEVIAGgREgBahASAlnOGpKo+WlUnq+rL68auqKqHqurJ+Xz5HK+ququqVqvq8aq6bisXD8DibeQTyR8muemMsTuSHB1j7E1ydJ4nyc1J9s7HwSR3b84yAVhW5wzJGOMvknz7jOF9SQ7P48NJbl03fu9Y80iS7VV11WYtFoDl80p/R7JzjPHsPH4uyc55vCvJM+vmHZ9jP6aqDlbVsao6durUqVe4DAAWrf3L9jHGSDJewesOjTFWxhgrO3bs6C4DgAV5pSF5/vQtq/l8co6fSLJn3bzdcwyAi9QrDcmRJPvn8f4kD6wbv21+e+v6JC+tuwUGwEVo27kmVNXHk/xKkp+rquNJ/kuSDya5r6oOJHk6ybvn9AeT3JJkNcn3krxvC9YMwBI5Z0jGGO/9CZduPMvckeT27qIAuHD4y3YAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgJZzhqSq9lTVw1X11ar6SlW9f45fUVUPVdWT8/nyOV5VdVdVrVbV41V13Vb/EAAszkY+kbyc5LfHGG9Kcn2S26vqTUnuSHJ0jLE3ydF5niQ3J9k7HweT3L3pqwZgaZwzJGOMZ8cYX5jH303yRJJdSfYlOTynHU5y6zzel+TeseaRJNur6qpNXzkAS+G8fkdSVVcneUuSR5PsHGM8Oy89l2TnPN6V5Jl1Lzs+x878tw5W1bGqOnbq1KnzXDYAy2LDIamqn03yx0k+MMb4zvprY4yRZJzPG48xDo0xVsYYKzt27DiflwKwRDYUkqp6TdYi8rExxp/M4edP37Kazyfn+Ikke9a9fPccA+AitJFvbVWSe5I8Mcb4/XWXjiTZP4/3J3lg3fht89tb1yd5ad0tMAAuMts2MOftSf5tki9V1WNz7D8m+WCS+6rqQJKnk7x7XnswyS1JVpN8L8n7NnXFACyVc4ZkjPF/ktRPuHzjWeaPJLc31wXABcJftgPQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC1CAkCLkADQIiQAtAgJAC3nDElVva6qPltVf1lVX6mq35vjb6iqR6tqtao+WVWXzvHXzvPVef3qrf0RAFikjXwi+fskN4wx3pzk2iQ3VdX1ST6U5M4xxhuTvJDkwJx/IMkLc/zOOQ+Ai9Q5QzLW/M08fc18jCQ3JLl/jh9Ocus83jfPM6/fWFW1aSsGYKls6HckVXVJVT2W5GSSh5J8PcmLY4yX55TjSXbN411JnkmSef2lJFee5d88WFXHqurYqVOnej8FAAuzoZCMMf5hjHFtkt1J3prkmu4bjzEOjTFWxhgrO3bs6P5zACzIeX1ra4zxYpKHk7wtyfaq2jYv7U5yYh6fSLInSeb11yf51qasFoCls5Fvbe2oqu3z+GeSvCPJE1kLyrvmtP1JHpjHR+Z55vXPjDHGZi4agOWx7dxTclWSw1V1SdbCc98Y49NV9dUkn6iq/5rki0numfPvSfI/qmo1ybeTvGcL1g3AkjhnSMYYjyd5y1nGn8ra70vOHP+7JL++KasDYOn5y3YAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgBYhAaBFSABoERIAWoQEgJYNh6SqLqmqL1bVp+f5G6rq0aparapPVtWlc/y183x1Xr96a5YOwDI4n08k70/yxLrzDyW5c4zxxiQvJDkwxw8keWGO3znnAXCR2lBIqmp3kl9N8pF5XkluSHL/nHI4ya3zeN88z7x+45wPwEVoo59I/iDJ7yT5x3l+ZZIXxxgvz/PjSXbN411JnkmSef2lOf9HVNXBqjpWVcdOnTr1CpcPwKKdMyRV9WtJTo4xPr+ZbzzGODTGWBljrOzYsWMz/2kAXkXbNjDn7UneWVW3JHldkn+W5MNJtlfVtvmpY3eSE3P+iSR7khyvqm1JXp/kW5u+cgCWwjk/kYwxfneMsXuMcXWS9yT5zBjjN5I8nORdc9r+JA/M4yPzPPP6Z8YYY1NXDcDS6PwdyX9I8ltVtZq134HcM8fvSXLlHP+tJHf0lgjAMtvIra0fGGP8eZI/n8dPJXnrWeb8XZJf34S1AXAB8JftALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAy4ZCUlXfqKovVdVjVXVsjl1RVQ9V1ZPz+fI5XlV1V1WtVtXjVXXdVv4AACzW+Xwi+ddjjGvHGCvz/I4kR8cYe5McnedJcnOSvfNxMMndm7VYAJZP59bWviSH5/HhJLeuG793rHkkyfaquqrxPgAssY2GZCT5X1X1+ao6OMd2jjGencfPJdk5j3cleWbda4/PsR9RVQer6lhVHTt16tQrWDoAy2DbBuf98hjjRFX98yQPVdVfrb84xhhVNc7njccYh5IcSpKVlZXzei0Ay2NDn0jGGCfm88kkn0ry1iTPn75lNZ9PzuknkuxZ9/LdcwyAi9A5Q1JVl1XVPz19nOTfJPlykiNJ9s9p+5M8MI+PJLltfnvr+iQvrbsFBsBFZiO3tnYm+VRVnZ7/P8cYf1pVn0tyX1UdSPJ0knfP+Q8muSXJapLvJXnfpq8agKVxzpCMMZ5K8uazjH8ryY1nGR9Jbt+U1QGw9PxlOwAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAi5AA0CIkALQICQAtQgJAy4ZCUlXbq+r+qvqrqnqiqt5WVVdU1UNV9eR8vnzOraq6q6pWq+rxqrpua38EABZpo59IPpzkT8cY1yR5c5InktyR5OgYY2+So/M8SW5Osnc+Dia5e1NXDMBSOWdIqur1Sf5VknuSZIzx/THGi0n2JTk8px1Ocus83pfk3rHmkSTbq+qqTV85AEthI59I3pDkVJL/XlVfrKqPVNVlSXaOMZ6dc55LsnMe70ryzLrXH59jP6KqDlbVsao6durUqVf+EwCwUBsJybYk1yW5e4zxliR/mx/exkqSjDFGknE+bzzGODTGWBljrOzYseN8XgrAEtlISI4nOT7GeHSe35+1sDx/+pbVfD45r59Ismfd63fPMQAuQucMyRjjuSTPVNUvzKEbk3w1yZEk++fY/iQPzOMjSW6b3966PslL626BAXCR2bbBef8+yceq6tIkTyV5X9YidF9VHUjydJJ3z7kPJrklyWqS7825AFykNhSSMcZjSVbOcunGs8wdSW5vrguAC4S/bAegRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFqEBIAWIQGgRUgAaBESAFrOGZKq+oWqemzd4ztV9YGquqKqHqqqJ+fz5XN+VdVdVbVaVY9X1XVb/2MAsCjnDMkY42tjjGvHGNcm+aUk30vyqSR3JDk6xtib5Og8T5Kbk+ydj4NJ7t6KhQOwHM731taNSb4+xng6yb4kh+f44SS3zuN9Se4dax5Jsr2qrtqU1QKwdM43JO9J8vF5vHOM8ew8fi7Jznm8K8kz615zfI79iKo6WFXHqurYqVOnznMZACyLDYekqi5N8s4kf3TmtTHGSDLO543HGIfGGCtjjJUdO3acz0sBWCLn84nk5iRfGGM8P8+fP33Laj6fnOMnkuxZ97rdcwyAi9D5hOS9+eFtrSQ5kmT/PN6f5IF147fNb29dn+SldbfAALjIbNvIpKq6LMk7kvzmuuEPJrmvqg4keTrJu+f4g0luSbKatW94vW/TVgvA0tlQSMYYf5vkyjPGvpW1b3GdOXckuX1TVgfA0vOX7QC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC0CAkALUICQIuQANAiJAC01Bhj0WtIVX03ydcWvY4LyM8l+eaiF3GBsFfnx35t3E/bXv2LMcaOs13Y9mqv5Cf42hhjZdGLuFBU1TH7tTH26vzYr42zVz/k1hYALUICQMuyhOTQohdwgbFfG2evzo/92jh7NS3FL9sBuHAtyycSAC5QQgJAy8JDUlU3VdXXqmq1qu5Y9HqWQVV9tKpOVtWX141dUVUPVdWT8/nyOV5Vddfcv8er6rrFrfzVV1V7qurhqvpqVX2lqt4/x+3XGarqdVX12ar6y7lXvzfH31BVj849+WRVXTrHXzvPV+f1qxe5/kWoqkuq6otV9el5bq/OYqEhqapLkvy3JDcneVOS91bVmxa5piXxh0luOmPsjiRHxxh7kxyd58na3u2dj4NJ7n6V1rgsXk7y22OMNyW5Psnt8z9D9uvH/X2SG8YYb05ybZKbqur6JB9KcucY441JXkhyYM4/kOSFOX7nnPfT5v1Jnlh3bq/OYtGfSN6aZHWM8dQY4/tJPpFk34LXtHBjjL9I8u0zhvclOTyPDye5dd34vWPNI0m2V9VVr85KF2+M8ewY4wvz+LtZ+y/9rtivHzN/5r+Zp6+Zj5HkhiT3z/Ez9+r0Ht6f5MaqqldpuQtXVbuT/GqSj8zzir06q0WHZFeSZ9adH59j/LidY4xn5/FzSXbOY3s4zdsJb0nyaOzXWc1bNY8lOZnkoSRfT/LiGOPlOWX9fvxgr+b1l5Jc+equeKH+IMnvJPnHeX5l7NVZLTokvAJj7Tvbvre9TlX9bJI/TvKBMcZ31l+zXz80xviHMca1SXZn7Y7ANQte0lKqql9LcnKM8flFr+VCsOiQnEiyZ9357jnGj3v+9C2Y+Xxyjv/U72FVvSZrEfnYGONP5rD9+v8YY7yY5OEkb8va7b3T/7976/fjB3s1r78+ybde5aUuytuTvLOqvpG1W+43JPlw7NVZLTokn0uyd34T4tIk70lyZMFrWlZHkuyfx/uTPLBu/Lb5baTrk7y07pbORW/eh74nyRNjjN9fd8l+naGqdlTV9nn8M0nekbXfKT2c5F1z2pl7dXoP35XkM+On5C+Yxxi/O8bYPca4Omv/u/SZMcZvxF6d3RhjoY8ktyT566zdq/1Pi17PMjySfDzJs0n+b9buwx7I2v3Wo0meTPK/k1wx51bWvvn29SRfSrKy6PW/ynv1y1m7bfV4ksfm4xb7dda9+sUkX5x79eUk/3mO/3ySzyZZTfJHSV47x183z1fn9Z9f9M+woH37lSSftlc/+eH/IgWAlkXf2gLgAickALQICQAtQgJAi5AA0CIkALQICQAt/w8DXrcS51YvkgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 480.24x799.92 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"Templates for exquisite corpses\"\"\"\n",
    "fragment_templates = [\n",
    "        ('pronoun_be', 'pronoun_be'),\n",
    "        ('pronoun_has', 'pronoun_has'),\n",
    "        ('pronoun_will', 'pronoun_will')\n",
    "]\n",
    "def fragments_for_time(time, fragments, templates):\n",
    "    \"\"\"time: tuple of (hour_string, min_string) or (hour_string) for special times.\n",
    "    :ret: tuple of fragment objects (frag1, frag2)\n",
    "    \"\"\"\n",
    "    \n",
    "    import numpy as np\n",
    "    import random\n",
    "    if len(time) == 1:\n",
    "        # Special time -- noon or midnight\n",
    "        frag1 = random.choice(fragments['special'][time[0]]['special'])\n",
    "        return frag1, None\n",
    "    else:\n",
    "        t1, t2 = time\n",
    "        frag1, frag2 = None, None\n",
    "        while (frag1 is None or frag2 is None):\n",
    "            try:\n",
    "                distribution = np.array([\n",
    "                    len(fragments['minutes'][t2][template2])\n",
    "                    for (template1, template2) in templates\n",
    "                ])\n",
    "\n",
    "                distribution = distribution / np.sum(distribution)\n",
    "                template_ind = np.random.choice(a=len(templates), p=distribution)\n",
    "                template1, template2 = templates[template_ind]\n",
    "                frag1 = random.choice(fragments['hours'][t1][template1])\n",
    "                frag1 = frag1[0].lower() + frag1[1:]\n",
    "                frag2 = random.choice(fragments['minutes'][t2][template2])\n",
    "                if frag1 is None or frag2 is None:\n",
    "                    assert False\n",
    "            except:\n",
    "                continue\n",
    "            return (frag1, frag2)\n",
    "\n",
    "\"\"\"Demo\"\"\"\n",
    "import random\n",
    "time_string = (random.choice(list(times['hours'].keys())),\n",
    "            random.choice(list(times['minutes'].keys())))\n",
    "frags = fragments_for_time(time_string, fragments, fragment_templates)\n",
    "\n",
    "print(time_string)\n",
    "frag1, frag2 = frags\n",
    "print(frag1)\n",
    "print(frag2)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.figure(figsize = (6.67, 11.11))\n",
    "plt.imshow(img,interpolation='nearest')\n",
    "\n",
    "test_time = [random.choice(list(times['special'].keys()))]\n",
    "frag1, frag2 = fragments_for_time(test_time, fragments, fragment_templates)\n",
    "print(test_time)\n",
    "print(frag1)\n",
    "\n",
    "print(frag2)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"TODO: render.\"\"\"\n",
    "def render(time_string, fragments, fragment_templates):\n",
    "    frag1, frag2 = fragments_for_time(time_string, fragments, fragment_templates)\n",
    "    CANVAS_SIZE = (480, 800)\n",
    "    from PIL import Image, ImageDraw, ImageFont\n",
    "    import os\n",
    "    fonts_path = os.path.realpath('./fonts/')\n",
    "    font = ImageFont.truetype(os.path.join(fonts_path, 'franklin_gothic_medium.ttf'), 20)\n",
    "\n",
    "    canvas = Image.new('RGB', CANVAS_SIZE, color = 'white')\n",
    "    if frag2 is None:\n",
    "        continue\n",
    "    else:\n",
    "        \n",
    "    return canvas\n",
    "\"\"\"Demo image\"\"\"\n",
    "img = render(time_string, fragments, fragment_templates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}