gitcount-tables.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PySAL Change Log Statistics: Table Generation\n",
    "\n",
    "This notebook generates the summary statistics for use in the 6-month releases of PySAL, which is now (2017-07) a meta package. \n",
    "\n",
    "It assumes the subpackages have been git cloned in a directory below the location of this notebook. It also requires network connectivity for some of the reporting.\n",
    "\n",
    "Run this notebook after `gitcount.ipynb`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import json\n",
    "import re\n",
    "import sys\n",
    "import pandas\n",
    "import subprocess\n",
    "from subprocess import check_output\n",
    "\n",
    "import yaml\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "from dateutil.parser import parse\n",
    "import pytz\n",
    "\n",
    "utc=pytz.UTC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta\n",
    "from time import sleep\n",
    "from subprocess import check_output\n",
    "try:\n",
    "    from urllib import urlopen\n",
    "except:\n",
    "    from urllib.request import urlopen\n",
    "\n",
    "import ssl\n",
    "import yaml\n",
    "\n",
    "context = ssl._create_unverified_context()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('packages.yml') as package_file:\n",
    "    packages = yaml.load(package_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "CWD = os.path.abspath(os.path.curdir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/serge/Dropbox/p/pysal/src/pysal'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CWD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our last main release was `2017-11-03`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2017, 11, 3, 0, 0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_date = '2017-11-03'\n",
    "since_date = '--since=\"{start}\"'.format(start=start_date)\n",
    "since_date\n",
    "since = datetime.strptime(start_date+\" 0:0:0\", \"%Y-%m-%d %H:%M:%S\")\n",
    "since"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "issue_details = pickle.load( open( \"issue_details.p\", \"rb\" ) )\n",
    "pull_details = pickle.load( open( \"pull_details.p\", \"rb\" ) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get dates of tags\n",
    "with open('subtags', 'r') as tag_name:\n",
    "        tags = tag_name.readlines()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "libpysal v4.0.1 2018-10-27\n",
      "esda v2.0.1 2018-11-04\n",
      "giddy v2.0.0 2018-08-26\n",
      "inequality v1.0.0 2018-10-31\n",
      "pointpats v2.0.0 2018-11-13\n",
      "spaghetti v1.1.0 2018-10-31\n",
      "mapclassify v2.0.1 2018-10-28\n",
      "spreg v1.0.4 2018-08-24\n",
      "spglm v1.0.6 2018-10-31\n",
      "spint v1.0.5 2019-01-04\n",
      "splot v1.0.0 2018-11-30\n",
      "mgwr v2.0.2 2019-01-05\n",
      "spvcm v0.2.1 2019-01-04\n"
     ]
    }
   ],
   "source": [
    "tag_dates = {}\n",
    "#root = '/home/serge/Dropbox/p/pysal/src/pysal/tmp/'\n",
    "root = CWD + \"/tmp/\"\n",
    "for record in tags:\n",
    "    pkg, tag = record.strip().split()\n",
    "    tag = tag.split('/')[-1]\n",
    "    pkdir = root+pkg\n",
    "    cmd = \"git log -1 --format=%ai {tag}\".format(tag=tag)\n",
    "    os.chdir(pkdir)\n",
    "    #print(cmd)\n",
    "    result = subprocess.run(cmd, check=True, shell=True, stdout=subprocess.PIPE)\n",
    "    tag_string = result.stdout.decode('utf-8')\n",
    "    tag_date = tag_string.split()[0]\n",
    "    tag_dates[pkg] = tag_date\n",
    "    print(pkg, tag, tag_date)\n",
    "\n",
    "os.chdir(CWD)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "libpysal 83 79\n",
      "libpysal 61 57\n",
      "esda 33 29\n",
      "esda 22 19\n",
      "giddy 58 38\n",
      "giddy 47 31\n",
      "inequality 11 6\n",
      "inequality 9 5\n",
      "pointpats 15 10\n",
      "pointpats 14 10\n",
      "spaghetti 205 94\n",
      "spaghetti 101 34\n",
      "mapclassify 15 9\n",
      "mapclassify 13 8\n",
      "spreg 11 8\n",
      "spreg 7 6\n",
      "spglm 15 12\n",
      "spglm 11 9\n",
      "spint 18 14\n",
      "spint 14 11\n",
      "splot 33 27\n",
      "splot 25 21\n",
      "mgwr 41 39\n",
      "mgwr 26 24\n",
      "spvcm 4 3\n",
      "spvcm 2 1\n"
     ]
    }
   ],
   "source": [
    "# get issues for a package and filter on tag date\n",
    "\n",
    "\n",
    "for pkg in tag_dates.keys():\n",
    "    issues = issue_details[pkg]\n",
    "    tag_date = utc.localize(parse(tag_dates[pkg]))\n",
    "    keep = []\n",
    "    for issue in issues:\n",
    "        closed = parse(issue['closed_at'])\n",
    "        if closed <= tag_date:\n",
    "            keep.append(issue)\n",
    "    print(pkg, len(issues), len(keep))\n",
    "    issue_details[pkg] = keep\n",
    "    keep = []\n",
    "    pulls = pull_details[pkg]\n",
    "    for pull in pulls:\n",
    "        closed = parse(pull['closed_at'])\n",
    "        if closed <= tag_date:\n",
    "            keep.append(pull)\n",
    "    print(pkg, len(pulls), len(keep)) \n",
    "    pull_details[pkg] = keep\n",
    "        \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "libpysal 276 276\n",
      "esda 80 80\n",
      "giddy 137 134\n",
      "inequality 36 32\n",
      "pointpats 40 40\n",
      "spaghetti 282 282\n",
      "mapclassify 48 48\n",
      "splot 247 242\n",
      "spreg 57 57\n",
      "spglm 70 69\n",
      "spint 63 62\n",
      "mgwr 245 245\n",
      "spvcm 55 55\n"
     ]
    }
   ],
   "source": [
    "# commits\n",
    "cmd = ['git', 'log', '--oneline', since_date]\n",
    "\n",
    "activity = {}\n",
    "total_commits = 0\n",
    "for package in packages:\n",
    "    subpackages = packages[package].split()\n",
    "    for subpackage in subpackages:\n",
    "        tag_date = tag_dates[subpackage]\n",
    "        os.chdir(CWD)\n",
    "        os.chdir('tmp/{subpackage}'.format(subpackage=subpackage))\n",
    "        cmd_until = cmd + ['--until=\"{tag_date}\"'.format(tag_date=tag_date)]\n",
    "        ncommits = len(check_output(cmd_until).splitlines())\n",
    "        ncommits_total = len(check_output(cmd).splitlines())\n",
    "        print(subpackage, ncommits_total, ncommits)\n",
    "        total_commits += ncommits\n",
    "        activity[subpackage] = ncommits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['git', 'log', '--oneline', '--since=\"2017-11-03\"', '--until=\"2019-01-04\"']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cmd_until"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "identities = {'Levi John Wolf': ('ljwolf', 'Levi John Wolf'),\n",
    "              'Serge Rey': ('Serge Rey', 'Sergio Rey', 'sjsrey', 'serge'),\n",
    "              'Wei Kang': ('Wei Kang', 'weikang9009'),\n",
    "              'Dani Arribas-Bel': ('Dani Arribas-Bel', 'darribas')\n",
    "}\n",
    "\n",
    "def regularize_identity(string):\n",
    "    string = string.decode()\n",
    "    for name, aliases in identities.items():\n",
    "        for alias in aliases:\n",
    "            if alias in string:\n",
    "                string = string.replace(alias, name)\n",
    "    if len(string.split(' '))>1:\n",
    "        string = string.title()\n",
    "    return string.lstrip('* ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_cmd = ['git', 'log', '--format=* %aN', since_date]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "author_cmd.append('blank')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['git', 'log', '--format=* %aN', '--since=\"2017-11-03\"', 'blank']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_cmd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "authors_global = set()\n",
    "authors = {}\n",
    "global_counter = Counter()\n",
    "counters = dict()\n",
    "cmd = ['git', 'log', '--oneline', since_date]\n",
    "total_commits = 0\n",
    "activity = {}\n",
    "for package in packages:\n",
    "    subpackages = packages[package].split()\n",
    "    for subpackage in subpackages:\n",
    "        os.chdir(CWD)\n",
    "        os.chdir('tmp/{subpackage}'.format(subpackage=subpackage))\n",
    "        ncommits = len(check_output(cmd).splitlines())\n",
    "        \n",
    "        \n",
    "        tag_date = tag_dates[subpackage]\n",
    "        author_cmd[-1] = '--until=\"{tag_date}\"'.format(tag_date=tag_date)\n",
    "        #cmd_until = cmd + ['--until=\"{tag_date}\"'.format(tag_date=tag_date)]\n",
    "\n",
    "        \n",
    "        all_authors = check_output(author_cmd).splitlines()\n",
    "        counter = Counter([regularize_identity(author) for author in all_authors])\n",
    "        global_counter += counter\n",
    "        counters.update({'.'.join((package,subpackage)): counter})\n",
    "        unique_authors = sorted(set(all_authors))\n",
    "        authors[subpackage] =  unique_authors\n",
    "        authors_global.update(unique_authors)\n",
    "        total_commits += ncommits\n",
    "        activity[subpackage] = ncommits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{b'* Dani Arribas-Bel',\n",
       " b'* Hu Shao',\n",
       " b'* James Gaboardi',\n",
       " b'* Levi John Wolf',\n",
       " b'* Philip Kahn',\n",
       " b'* Serge Rey',\n",
       " b'* Sergio Rey',\n",
       " b'* Stefanie Lumnitz',\n",
       " b'* Taylor Oshan',\n",
       " b'* Wei Kang',\n",
       " b'* Ziqi Li',\n",
       " b'* eli knaap',\n",
       " b'* jsignell',\n",
       " b'* ljwolf',\n",
       " b'* serge',\n",
       " b'* thequackdaddy',\n",
       " b'* weikang9009'}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authors_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'libpysal': 276,\n",
       " 'esda': 80,\n",
       " 'giddy': 137,\n",
       " 'inequality': 36,\n",
       " 'pointpats': 40,\n",
       " 'spaghetti': 282,\n",
       " 'mapclassify': 48,\n",
       " 'splot': 247,\n",
       " 'spreg': 57,\n",
       " 'spglm': 70,\n",
       " 'spint': 63,\n",
       " 'mgwr': 245,\n",
       " 'spvcm': 55}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "activity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lib.libpysal': Counter({'Serge Rey': 101,\n",
       "          'Levi John Wolf': 112,\n",
       "          'Wei Kang': 18,\n",
       "          'James Gaboardi': 8,\n",
       "          'Eli Knaap': 12,\n",
       "          'Dani Arribas-Bel': 19,\n",
       "          'Taylor Oshan': 4,\n",
       "          'Stefanie Lumnitz': 2}),\n",
       " 'explore.esda': Counter({'Serge Rey': 50,\n",
       "          'James Gaboardi': 1,\n",
       "          'Wei Kang': 8,\n",
       "          'Levi John Wolf': 11,\n",
       "          'Stefanie Lumnitz': 10}),\n",
       " 'explore.giddy': Counter({'Wei Kang': 103,\n",
       "          'Stefanie Lumnitz': 10,\n",
       "          'Serge Rey': 18,\n",
       "          'Levi John Wolf': 1,\n",
       "          'Eli Knaap': 2}),\n",
       " 'explore.inequality': Counter({'Wei Kang': 11,\n",
       "          'Serge Rey': 20,\n",
       "          'Levi John Wolf': 1}),\n",
       " 'explore.pointpats': Counter({'Wei Kang': 26,\n",
       "          'Hu Shao': 5,\n",
       "          'Levi John Wolf': 7,\n",
       "          'Serge Rey': 2}),\n",
       " 'explore.spaghetti': Counter({'James Gaboardi': 278,\n",
       "          'Levi John Wolf': 3,\n",
       "          'Wei Kang': 1}),\n",
       " 'viz.mapclassify': Counter({'Wei Kang': 23,\n",
       "          'Serge Rey': 22,\n",
       "          'Levi John Wolf': 3}),\n",
       " 'viz.splot': Counter({'Dani Arribas-Bel': 6,\n",
       "          'Stefanie Lumnitz': 218,\n",
       "          'Levi John Wolf': 9,\n",
       "          'Serge Rey': 7,\n",
       "          'Thequackdaddy': 1,\n",
       "          'Jsignell': 1}),\n",
       " 'model.spreg': Counter({'Levi John Wolf': 40, 'Serge Rey': 17}),\n",
       " 'model.spglm': Counter({'Taylor Oshan': 60,\n",
       "          'Wei Kang': 1,\n",
       "          'James Gaboardi': 7,\n",
       "          'Levi John Wolf': 1}),\n",
       " 'model.spint': Counter({'Wei Kang': 4,\n",
       "          'Levi John Wolf': 9,\n",
       "          'Taylor Oshan': 42,\n",
       "          'James Gaboardi': 7}),\n",
       " 'model.mgwr': Counter({'Levi John Wolf': 13,\n",
       "          'James Gaboardi': 12,\n",
       "          'Wei Kang': 30,\n",
       "          'Taylor Oshan': 137,\n",
       "          'Philip Kahn': 2,\n",
       "          'Ziqi Li': 51}),\n",
       " 'model.spvcm': Counter({'Levi John Wolf': 52,\n",
       "          'James Gaboardi': 1,\n",
       "          'Serge Rey': 2})}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lib.libpysal': Counter({'Serge Rey': 101,\n",
       "          'Levi John Wolf': 112,\n",
       "          'Wei Kang': 18,\n",
       "          'James Gaboardi': 8,\n",
       "          'Eli Knaap': 12,\n",
       "          'Dani Arribas-Bel': 19,\n",
       "          'Taylor Oshan': 4,\n",
       "          'Stefanie Lumnitz': 2}),\n",
       " 'explore.esda': Counter({'Serge Rey': 50,\n",
       "          'James Gaboardi': 1,\n",
       "          'Wei Kang': 8,\n",
       "          'Levi John Wolf': 11,\n",
       "          'Stefanie Lumnitz': 10}),\n",
       " 'explore.giddy': Counter({'Wei Kang': 103,\n",
       "          'Stefanie Lumnitz': 10,\n",
       "          'Serge Rey': 18,\n",
       "          'Levi John Wolf': 1,\n",
       "          'Eli Knaap': 2}),\n",
       " 'explore.inequality': Counter({'Wei Kang': 11,\n",
       "          'Serge Rey': 20,\n",
       "          'Levi John Wolf': 1}),\n",
       " 'explore.pointpats': Counter({'Wei Kang': 26,\n",
       "          'Hu Shao': 5,\n",
       "          'Levi John Wolf': 7,\n",
       "          'Serge Rey': 2}),\n",
       " 'explore.spaghetti': Counter({'James Gaboardi': 278,\n",
       "          'Levi John Wolf': 3,\n",
       "          'Wei Kang': 1}),\n",
       " 'viz.mapclassify': Counter({'Wei Kang': 23,\n",
       "          'Serge Rey': 22,\n",
       "          'Levi John Wolf': 3}),\n",
       " 'viz.splot': Counter({'Dani Arribas-Bel': 6,\n",
       "          'Stefanie Lumnitz': 218,\n",
       "          'Levi John Wolf': 9,\n",
       "          'Serge Rey': 7,\n",
       "          'Thequackdaddy': 1,\n",
       "          'Jsignell': 1}),\n",
       " 'model.spreg': Counter({'Levi John Wolf': 40, 'Serge Rey': 17}),\n",
       " 'model.spglm': Counter({'Taylor Oshan': 60,\n",
       "          'Wei Kang': 1,\n",
       "          'James Gaboardi': 7,\n",
       "          'Levi John Wolf': 1}),\n",
       " 'model.spint': Counter({'Wei Kang': 4,\n",
       "          'Levi John Wolf': 9,\n",
       "          'Taylor Oshan': 42,\n",
       "          'James Gaboardi': 7}),\n",
       " 'model.mgwr': Counter({'Levi John Wolf': 13,\n",
       "          'James Gaboardi': 12,\n",
       "          'Wei Kang': 30,\n",
       "          'Taylor Oshan': 137,\n",
       "          'Philip Kahn': 2,\n",
       "          'Ziqi Li': 51}),\n",
       " 'model.spvcm': Counter({'Levi John Wolf': 52,\n",
       "          'James Gaboardi': 1,\n",
       "          'Serge Rey': 2})}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'url': 'https://api.github.com/repos/pysal/spvcm/issues/6',\n",
       " 'repository_url': 'https://api.github.com/repos/pysal/spvcm',\n",
       " 'labels_url': 'https://api.github.com/repos/pysal/spvcm/issues/6/labels{/name}',\n",
       " 'comments_url': 'https://api.github.com/repos/pysal/spvcm/issues/6/comments',\n",
       " 'events_url': 'https://api.github.com/repos/pysal/spvcm/issues/6/events',\n",
       " 'html_url': 'https://github.com/pysal/spvcm/pull/6',\n",
       " 'id': 381923508,\n",
       " 'node_id': 'MDExOlB1bGxSZXF1ZXN0MjMxNzY1NDM4',\n",
       " 'number': 6,\n",
       " 'title': 'enh: swap libpysal in for pysal',\n",
       " 'user': {'login': 'sjsrey',\n",
       "  'id': 118042,\n",
       "  'node_id': 'MDQ6VXNlcjExODA0Mg==',\n",
       "  'avatar_url': 'https://avatars1.githubusercontent.com/u/118042?v=4',\n",
       "  'gravatar_id': '',\n",
       "  'url': 'https://api.github.com/users/sjsrey',\n",
       "  'html_url': 'https://github.com/sjsrey',\n",
       "  'followers_url': 'https://api.github.com/users/sjsrey/followers',\n",
       "  'following_url': 'https://api.github.com/users/sjsrey/following{/other_user}',\n",
       "  'gists_url': 'https://api.github.com/users/sjsrey/gists{/gist_id}',\n",
       "  'starred_url': 'https://api.github.com/users/sjsrey/starred{/owner}{/repo}',\n",
       "  'subscriptions_url': 'https://api.github.com/users/sjsrey/subscriptions',\n",
       "  'organizations_url': 'https://api.github.com/users/sjsrey/orgs',\n",
       "  'repos_url': 'https://api.github.com/users/sjsrey/repos',\n",
       "  'events_url': 'https://api.github.com/users/sjsrey/events{/privacy}',\n",
       "  'received_events_url': 'https://api.github.com/users/sjsrey/received_events',\n",
       "  'type': 'User',\n",
       "  'site_admin': False},\n",
       " 'labels': [],\n",
       " 'state': 'closed',\n",
       " 'locked': False,\n",
       " 'assignee': None,\n",
       " 'assignees': [],\n",
       " 'milestone': None,\n",
       " 'comments': 13,\n",
       " 'created_at': '2018-11-18T03:08:27Z',\n",
       " 'updated_at': '2019-01-04T19:22:46Z',\n",
       " 'closed_at': '2019-01-04T19:22:46Z',\n",
       " 'author_association': 'MEMBER',\n",
       " 'pull_request': {'url': 'https://api.github.com/repos/pysal/spvcm/pulls/6',\n",
       "  'html_url': 'https://github.com/pysal/spvcm/pull/6',\n",
       "  'diff_url': 'https://github.com/pysal/spvcm/pull/6.diff',\n",
       "  'patch_url': 'https://github.com/pysal/spvcm/pull/6.patch'},\n",
       " 'body': ''}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "issues[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tag(title, level=\"##\", as_string=True):\n",
    "    words = title.split()\n",
    "    tag = \"-\".join([word.lower() for word in words])\n",
    "    heading = level+\" \"+title\n",
    "    line = \"\\n\\n<a name=\\\"{}\\\"></a>\".format(tag)\n",
    "    lines = [line]\n",
    "    lines.append(heading)\n",
    "    if as_string:\n",
    "        return \"\\n\".join(lines)\n",
    "    else:\n",
    "        return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "subs = issue_details.keys()\n",
    "table = []\n",
    "txt = []\n",
    "lines = get_tag(\"Changes by Package\", as_string=False)\n",
    "\n",
    "for sub in subs:\n",
    "    total= issue_details[sub]\n",
    "    pr = pull_details[sub]\n",
    "    \n",
    "    row = [sub, activity[sub], len(total), len(pr)]\n",
    "    table.append(row)\n",
    "    #line = \"\\n<a name=\\\"{sub}\\\"></a>\".format(sub=sub)\n",
    "    #lines.append(line)\n",
    "    #line = \"### {sub}\".format(sub=sub)\n",
    "    #lines.append(line)\n",
    "    lines.extend(get_tag(sub.lower(), \"###\", as_string=False))\n",
    "    for issue in total:\n",
    "        url = issue['html_url']\n",
    "        title = issue['title']\n",
    "        number = issue['number']\n",
    "        line = \"* {title} [(#{number})]({url})\".format(title=title,\n",
    "                                                     number=number,\n",
    "                                                     url=url)\n",
    "        lines.append(line)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['libpysal', 276, 79, 57],\n",
       " ['esda', 80, 29, 19],\n",
       " ['giddy', 137, 38, 31],\n",
       " ['inequality', 36, 6, 5],\n",
       " ['pointpats', 40, 10, 10],\n",
       " ['spaghetti', 282, 94, 34],\n",
       " ['mapclassify', 48, 9, 8],\n",
       " ['splot', 247, 27, 21],\n",
       " ['spreg', 57, 8, 6],\n",
       " ['spglm', 70, 12, 9],\n",
       " ['spint', 63, 14, 11],\n",
       " ['mgwr', 245, 39, 24],\n",
       " ['spvcm', 55, 3, 1]]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(CWD)\n",
    "\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pandas.DataFrame(table, columns=['package', 'commits', 'total issues', 'pulls'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values(['commits','pulls'], ascending=False)\\\n",
    "  .to_html('./commit_table.html', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "package         libpysalesdagiddyinequalitypointpatsspaghettim...\n",
       "commits                                                      1636\n",
       "total issues                                                  368\n",
       "pulls                                                         236\n",
       "dtype: object"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributor_table = pandas.DataFrame.from_dict(counters).fillna(0).astype(int).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "contributor_table.to_html('./contributor_table.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>commits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Dani Arribas-Bel</th>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Eli Knaap</th>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hu Shao</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>James Gaboardi</th>\n",
       "      <td>314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Jsignell</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Levi John Wolf</th>\n",
       "      <td>262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Philip Kahn</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Serge Rey</th>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stefanie Lumnitz</th>\n",
       "      <td>240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Taylor Oshan</th>\n",
       "      <td>243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thequackdaddy</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wei Kang</th>\n",
       "      <td>225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ziqi Li</th>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  commits\n",
       "Dani Arribas-Bel       25\n",
       "Eli Knaap              14\n",
       "Hu Shao                 5\n",
       "James Gaboardi        314\n",
       "Jsignell                1\n",
       "Levi John Wolf        262\n",
       "Philip Kahn             2\n",
       "Serge Rey             239\n",
       "Stefanie Lumnitz      240\n",
       "Taylor Oshan          243\n",
       "Thequackdaddy           1\n",
       "Wei Kang              225\n",
       "Ziqi Li                51"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totals = contributor_table.sum(axis=0).T\n",
    "totals.sort_index().to_frame('commits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "totals = contributor_table.sum(axis=0).T\n",
    "totals.sort_index().to_frame('commits').to_html('./commits_by_person.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_commits = df.commits.sum()\n",
    "n_issues = df['total issues'].sum()\n",
    "n_pulls = df.pulls.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1636"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_commits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Overall, there were 719 commits that closed 240 issues, together with 105 pull requests across 12 packages since our last release on 2017-11-03.\n",
    "#('{0} Here is a really long '\n",
    "#           'sentence with {1}').format(3, 5))\n",
    "line = ('Overall, there were {n_commits} commits that closed {n_issues} issues,'  \n",
    "    ' together with {n_pulls} pull requests since our last release' \n",
    "        ' on {since_date}.\\n'.format(n_commits=n_commits, n_issues=n_issues,\n",
    "        n_pulls=n_pulls, since_date = start_date))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Overall, there were 1636 commits that closed 368 issues, together with 236 pull requests since our last release on 2017-11-03.\\n'"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## append html files to end of changes.md with tags for toc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('changes.md', 'w') as fout:\n",
    "    fout.write(line)\n",
    "    fout.write(\"\\n\".join(lines))\n",
    "    fout.write(get_tag(\"Summary Statistics\"))\n",
    "    \n",
    "    with open('commit_table.html') as table:\n",
    "        table_lines = table.readlines()\n",
    "        title = \"Package Activity\"\n",
    "        fout.write(get_tag(title,\"###\"))\n",
    "        fout.write(\"\\n\")\n",
    "        fout.write(\"\".join(table_lines))\n",
    "                \n",
    "    with open('commits_by_person.html') as table:\n",
    "        table_lines = table.readlines()\n",
    "        title = \"Contributor Activity\"\n",
    "        fout.write(get_tag(title,\"###\"))\n",
    "        fout.write(\"\\n\")\n",
    "        fout.write(\"\".join(table_lines))\n",
    "        \n",
    "    with open('contributor_table.html') as table:\n",
    "        table_lines = table.readlines()\n",
    "        title = \"Contributor by Package Activity\"\n",
    "        fout.write(get_tag(title,\"###\"))\n",
    "        fout.write(\"\\n\")\n",
    "        fout.write(\"\".join(table_lines))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}