Predicting crime.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "using DataFrames, PyPlot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load data\n",
    "crime = readtable(\"crime.csv\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# clean data\n",
    "counties = Set((crime[:County]))\n",
    "reporting_counties = String[]\n",
    "nonreporting_counties = String[]\n",
    "\n",
    "for county in counties\n",
    "    # println(\"$county reported for $(length(crime[crime[:County].==county,:County])) years.\")\n",
    "    if length(crime[crime[:County].==county,:County]) == 26\n",
    "        push!(reporting_counties, county)\n",
    "    else\n",
    "        push!(nonreporting_counties, county)\n",
    "    end\n",
    "end\n",
    "\n",
    "reported_crime_all_years = Array(Bool, size(crime,1))\n",
    "for (i,county) in enumerate(crime[:County])\n",
    "    reported_crime_all_years[i] = ~(county in nonreporting_counties)\n",
    "end\n",
    "\n",
    "crime = crime[reported_crime_all_years,:];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# just Tompkins county\n",
    "tompkins = crime[crime[:County].==\"Tompkins\",:];\n",
    "n = size(tompkins,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# predict crime in Tompkins county"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# how about just using the year?\n",
    "X = [tompkins[:Year].data ones(n)]\n",
    "y = tompkins[:Index_Count].data\n",
    "w = X\\y\n",
    "\n",
    "tompkins[:pred_linear] = X*w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf()\n",
    "plot(tompkins[:Year], tompkins[:Index_Count], \"ro\", label=\"Index_Count\"),\n",
    "plot(tompkins[:Year], tompkins[:pred_linear], \"b-\", label=\"linear model\")\n",
    "legend()\n",
    "xlabel(\"Year\")\n",
    "ylabel(\"Crime\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autoregressive models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# how about using the level of crime last year? (called ``lagged outcome'')\n",
    "X = [tompkins[:Index_Count].data[2:end] ones(n-1)]\n",
    "y = tompkins[:Index_Count].data[1:end-1]\n",
    "w = X\\y\n",
    "\n",
    "tompkins[:pred_ar1] = vcat(X*w, tompkins[:pred_linear][end]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf()\n",
    "plot(tompkins[:Year], tompkins[:Index_Count], \"ro\", label=\"Index_Count\"),\n",
    "plot(tompkins[:Year], tompkins[:pred_linear], \"b-\", label=\"linear model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1][1:end-1], \"y-\", label=\"AR1 model\")\n",
    "legend()\n",
    "xlabel(\"Year\")\n",
    "ylabel(\"Crime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# how about using the year *and* the level of crime last year? (called ``lagged outcome'')\n",
    "X = [tompkins[:Year].data[1:end-1] tompkins[:Index_Count].data[2:end] ones(n-1)]\n",
    "y = tompkins[:Index_Count].data[1:end-1]\n",
    "w = X\\y\n",
    "\n",
    "tompkins[:pred_ar1_lin] = vcat(X*w, tompkins[:pred_linear][end]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf()\n",
    "plot(tompkins[:Year], tompkins[:Index_Count], \"ro\", label=\"Index_Count\"),\n",
    "plot(tompkins[:Year], tompkins[:pred_linear], \"b-\", label=\"linear model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1][1:end-1], \"y-\", label=\"AR1 model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1_lin][1:end-1], \"g-\", label=\"AR + linear model\")\n",
    "legend()\n",
    "xlabel(\"Year\")\n",
    "ylabel(\"Crime\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Smoothed models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "α = 1\n",
    "n = length(tompkins[:Year])\n",
    "X = zeros(n,n)\n",
    "for i=1:n\n",
    "    for j=1:n\n",
    "        X[i,j] = exp(-(tompkins[:Year][i]-tompkins[:Year][j])^2)\n",
    "    end\n",
    "end\n",
    "y = tompkins[:Index_Count].data\n",
    "w = X\\y\n",
    "\n",
    "tompkins[:pred_smooth] = X*w;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf()\n",
    "plot(tompkins[:Year], tompkins[:Index_Count], \"ro\", label=\"Index_Count\"),\n",
    "plot(tompkins[:Year], tompkins[:pred_linear], \"b-\", label=\"linear model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1][1:end-1], \"y-\", label=\"AR1 model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1_lin][1:end-1], \"g-\", label=\"AR + linear model\")\n",
    "plot(tompkins[:Year], tompkins[:pred_smooth], \"c-\", label=\"smoothed model\")\n",
    "legend()\n",
    "xlabel(\"Year\")\n",
    "ylabel(\"Crime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "α = 1\n",
    "n = length(tompkins[:Year])\n",
    "nknots = Int(n/2)\n",
    "X = zeros(n,nknots)\n",
    "for i=1:n\n",
    "    for j=1:nknots\n",
    "        X[i,j] = exp(-(tompkins[:Year][i]-tompkins[:Year][2*j])^2)\n",
    "    end\n",
    "end\n",
    "X = [X ones(n)]\n",
    "y = tompkins[:Index_Count].data\n",
    "w = X\\y\n",
    "\n",
    "tompkins[:pred_smooth] = X*w;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf()\n",
    "plot(tompkins[:Year], tompkins[:Index_Count], \"ro\", label=\"Index_Count\"),\n",
    "plot(tompkins[:Year], tompkins[:pred_linear], \"b-\", label=\"linear model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1][1:end-1], \"y-\", label=\"AR1 model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1_lin][1:end-1], \"g-\", label=\"AR + linear model\")\n",
    "plot(tompkins[:Year], tompkins[:pred_smooth], \"c-\", label=\"smoothed model\")\n",
    "legend()\n",
    "xlabel(\"Year\")\n",
    "ylabel(\"Crime\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# regularize + smooth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "α = 1\n",
    "n = length(tompkins[:Year])\n",
    "X = zeros(n,n)\n",
    "for i=1:n\n",
    "    for j=1:n\n",
    "        X[i,j] = exp(-(tompkins[:Year][i]-tompkins[:Year][j])^2)\n",
    "    end\n",
    "end\n",
    "y = tompkins[:Index_Count].data - tompkins[:pred_linear]\n",
    "w = (X'*X + eye(n))\\X'*y\n",
    "\n",
    "tompkins[:pred_smooth_reg] = X*w + tompkins[:pred_linear];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf()\n",
    "plot(tompkins[:Year], tompkins[:Index_Count], \"ro\", label=\"Index_Count\"),\n",
    "plot(tompkins[:Year], tompkins[:pred_linear], \"b-\", label=\"linear model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1][1:end-1], \"y-\", label=\"AR1 model\")\n",
    "plot(tompkins[:Year][1:end-1], tompkins[:pred_ar1_lin][1:end-1], \"g-\", label=\"AR + linear model\")\n",
    "plot(tompkins[:Year], tompkins[:pred_smooth], \"c-\", label=\"smoothed model\")\n",
    "plot(tompkins[:Year], tompkins[:pred_smooth_reg], \"m-\", label=\"reg smoothed model\")\n",
    "legend()\n",
    "xlabel(\"Year\")\n",
    "ylabel(\"Crime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.5.0-rc3",
   "language": "julia",
   "name": "julia-0.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}