06_Data_Exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example with Notification Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to use this code you need to recorde your own data using \"Notification Log\". \n",
    "* Get \"Notification Log\" from https://play.google.com/store/apps/details?id=org.hcilab.projects.nlog\n",
    "* Record your data\n",
    "* export the data\n",
    "* change DATA_URL to yout datapath\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_URL = \"./dataset_4/EXPORT_FILE.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_PATH = \"./dataset_4/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  \n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "regex = re.compile('[^a-zA-Z]')\n",
    "fOut = open(DATA_URL)\n",
    "with open(DATA_URL) as f:\n",
    "    for l in f:\n",
    "        if (l.startswith(\"#\")) | (l.startswith(\"#\")) | (l.startswith(\"#\")):\n",
    "            if (fOut.closed == False):\n",
    "                fOut.close()\n",
    "            pathOut = '%s%s.txt' % (DATA_PATH, regex.sub('', l.lower()))\n",
    "            fOut = open(pathOut, 'w')\n",
    "        #print (l)\n",
    "        else:\n",
    "            fOut.write(l)  # python will convert \\n to os.linesep\n",
    "if (fOut.closed == False):\n",
    "    fOut.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfD = pd.read_json(DATA_PATH + 'device.txt', lines=True)\n",
    "dfD.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfP = pd.read_json(DATA_PATH + 'posted.txt', lines=True)\n",
    "dfP['postTime'] = pd.to_datetime(dfP['postTime'], unit='ms')\n",
    "dfP['systemTime'] = pd.to_datetime(dfP['systemTime'], unit='ms')\n",
    "dfP['when'] = pd.to_datetime(dfP['when'], unit='ms')\n",
    "dfP = dfP.set_index(\"systemTime\")\n",
    "dfP.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfR = pd.read_json(DATA_PATH + 'removed.txt', lines=True)\n",
    "dfR.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfP.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfP.offset.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfP['C'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfP.ringerMode.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfP['month'] = dfP.postTime.apply(lambda x: \"%d-%d\" % (x.year, x.month))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfP.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfP['month'] = pd.to_datetime(dfP['month'], format='%Y-%m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfP['date'] = dfP.postTime.apply(lambda x: x.date())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfP['weekday'] = dfP.postTime.apply(lambda x: x.weekday())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How many Mondays are in the dataset?\n",
    "dfP.drop_duplicates(\"date\").groupby(\"weekday\").date.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g1 = dfP.groupby(\"weekday\")\n",
    "(g1.C.sum()/60).plot(title=\"Avg Notivication per Day\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g1 = dfP.groupby(\"month\")\n",
    "(g1.C.sum()).plot(title=\"Count Notivication per Month\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g1 = dfP[dfP.packageName == \"com.whatsapp\"].groupby(\"month\")\n",
    "(g1.C.sum()/60).plot(label=\"whatsapp\")\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.google.android.gm\"].groupby(\"month\")\n",
    "(g1.C.sum()/60).plot(label=\"gmail\")\n",
    "\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.microsoft.office.onenote\"].groupby(\"month\")\n",
    "(g1.C.sum()/60).plot(label=\"Office Onenote\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g1 = dfP[dfP.packageName == \"com.whatsapp\"].groupby(\"weekday\")\n",
    "(g1.C.sum()/60).plot(label=\"whatsapp\")\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.google.android.gm\"].groupby(\"weekday\")\n",
    "(g1.C.sum()/60).plot(label=\"GMail\")\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.facebook.katana\"].groupby(\"weekday\")\n",
    "(g1.C.sum()/60).plot(label=\"facebook\")\n",
    "plt.legend()\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.facebook.orca\"].groupby(\"weekday\")\n",
    "(g1.C.sum()/60).plot(label=\"facebook M\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g1 = dfP[dfP.packageName == \"com.whatsapp\"].groupby(\"month\")\n",
    "(g1.C.sum()).plot(label=\"whatsapp\")\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.google.android.gm\"].groupby(\"month\")\n",
    "(g1.C.sum()).plot(label=\"GMail\")\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.facebook.katana\"].groupby(\"month\")\n",
    "(g1.C.sum()).plot(label=\"facebook\")\n",
    "plt.legend()\n",
    "\n",
    "g1 = dfP[dfP.packageName == \"com.facebook.orca\"].groupby(\"month\")\n",
    "(g1.C.sum()).plot(label=\"facebook M\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = dfP.groupby(\"weekday\")\n",
    "g1 = dfP[dfP.packageName == \"com.whatsapp\"].groupby(\"weekday\")\n",
    "(g1.C.sum()/60) / (g.C.sum()/60)*100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}