exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:08.477392",
     "start_time": "2017-02-14T14:32:08.473113"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "pd.set_option(\"display.notebook_repr_html\", False)\n",
    "pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:08.979453",
     "start_time": "2017-02-14T14:32:08.690088"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/markhashimoto/anaconda/lib/python3.5/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (90,105,119,120,123,124,132,134,135,138,139,147,149,150,153,154,162,164,165,168,169,177,179,183,184,192,194,198,199,207,209,213,214,224,237,239,244) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv(\"data/unimelb_training.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:38:02.050580",
     "start_time": "2017-02-14T14:38:02.019123"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8708 entries, 0 to 8707\n",
      "Columns: 252 entries, Grant.Application.ID to Unnamed: 251\n",
      "dtypes: float64(180), int64(2), object(70)\n",
      "memory usage: 16.7+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:09.101590",
     "start_time": "2017-02-14T14:32:09.010925"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grant.Application.ID                       8708\n",
      "Grant.Status                               8708\n",
      "Sponsor.Code                               7796\n",
      "Grant.Category.Code                        7796\n",
      "Contract.Value.Band...see.note.A           5145\n",
      "Start.date                                 8708\n",
      "RFCD.Code.1                                8074\n",
      "RFCD.Percentage.1                          8074\n",
      "RFCD.Code.2                                8074\n",
      "RFCD.Percentage.2                          8074\n",
      "RFCD.Code.3                                8074\n",
      "RFCD.Percentage.3                          8074\n",
      "RFCD.Code.4                                8074\n",
      "RFCD.Percentage.4                          8074\n",
      "RFCD.Code.5                                8074\n",
      "RFCD.Percentage.5                          8074\n",
      "SEO.Code.1                                 8044\n",
      "SEO.Percentage.1                           8044\n",
      "SEO.Code.2                                 8044\n",
      "SEO.Percentage.2                           8044\n",
      "SEO.Code.3                                 8044\n",
      "SEO.Percentage.3                           8044\n",
      "SEO.Code.4                                 8044\n",
      "SEO.Percentage.4                           8044\n",
      "SEO.Code.5                                 8044\n",
      "SEO.Percentage.5                           8044\n",
      "Person.ID.1                                8185\n",
      "Role.1                                     8610\n",
      "Year.of.Birth.1                            8181\n",
      "Country.of.Birth.1                         7428\n",
      "Home.Language.1                             798\n",
      "Dept.No..1                                 7838\n",
      "Faculty.No..1                              7835\n",
      "With.PHD.1                                 5113\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.1     7227\n",
      "Number.of.Successful.Grant.1               8185\n",
      "Number.of.Unsuccessful.Grant.1             8185\n",
      "A..1                                       8185\n",
      "A.1                                        8185\n",
      "B.1                                        8185\n",
      "C.1                                        8185\n",
      "Person.ID.2                                2989\n",
      "Role.2                                     4402\n",
      "Year.of.Birth.2                            2989\n",
      "Country.of.Birth.2                         2516\n",
      "Home.Language.2                             274\n",
      "Dept.No..2                                 2638\n",
      "Faculty.No..2                              2637\n",
      "With.PHD.2                                 1748\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.2     2504\n",
      "Number.of.Successful.Grant.2               2989\n",
      "Number.of.Unsuccessful.Grant.2             2989\n",
      "A..2                                       2989\n",
      "A.2                                        2989\n",
      "B.2                                        2989\n",
      "C.2                                        2989\n",
      "Person.ID.3                                1134\n",
      "Role.3                                     2191\n",
      "Year.of.Birth.3                            1134\n",
      "Country.of.Birth.3                          953\n",
      "Home.Language.3                              97\n",
      "Dept.No..3                                  979\n",
      "Faculty.No..3                               977\n",
      "With.PHD.3                                  653\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.3      939\n",
      "Number.of.Successful.Grant.3               1134\n",
      "Number.of.Unsuccessful.Grant.3             1134\n",
      "A..3                                       1134\n",
      "A.3                                        1134\n",
      "B.3                                        1134\n",
      "C.3                                        1134\n",
      "Person.ID.4                                 424\n",
      "Role.4                                     1156\n",
      "Year.of.Birth.4                             424\n",
      "Country.of.Birth.4                          373\n",
      "Home.Language.4                              31\n",
      "Dept.No..4                                  379\n",
      "Faculty.No..4                               379\n",
      "With.PHD.4                                  275\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.4      366\n",
      "Number.of.Successful.Grant.4                424\n",
      "Number.of.Unsuccessful.Grant.4              424\n",
      "A..4                                        424\n",
      "A.4                                         424\n",
      "B.4                                         424\n",
      "C.4                                         424\n",
      "Person.ID.5                                 156\n",
      "Role.5                                      568\n",
      "Year.of.Birth.5                             156\n",
      "Country.of.Birth.5                          140\n",
      "Home.Language.5                              14\n",
      "Dept.No..5                                    0\n",
      "Faculty.No..5                               142\n",
      "With.PHD.5                                  100\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.5      136\n",
      "Number.of.Successful.Grant.5                156\n",
      "Number.of.Unsuccessful.Grant.5              156\n",
      "A..5                                        156\n",
      "A.5                                         156\n",
      "B.5                                         156\n",
      "C.5                                         156\n",
      "Person.ID.6                                  70\n",
      "Role.6                                      310\n",
      "Year.of.Birth.6                              70\n",
      "Country.of.Birth.6                           57\n",
      "Home.Language.6                               4\n",
      "Dept.No..6                                   57\n",
      "Faculty.No..6                                57\n",
      "With.PHD.6                                   44\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.6       56\n",
      "Number.of.Successful.Grant.6                 70\n",
      "Number.of.Unsuccessful.Grant.6               70\n",
      "A..6                                         70\n",
      "A.6                                          70\n",
      "B.6                                          70\n",
      "C.6                                          70\n",
      "Person.ID.7                                  36\n",
      "Role.7                                       93\n",
      "Year.of.Birth.7                              36\n",
      "Country.of.Birth.7                           16\n",
      "Home.Language.7                               2\n",
      "Dept.No..7                                   33\n",
      "Faculty.No..7                                33\n",
      "With.PHD.7                                   25\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.7       32\n",
      "Number.of.Successful.Grant.7                 36\n",
      "Number.of.Unsuccessful.Grant.7               36\n",
      "A..7                                         36\n",
      "A.7                                          36\n",
      "B.7                                          36\n",
      "C.7                                          36\n",
      "Person.ID.8                                  12\n",
      "Role.8                                       53\n",
      "Year.of.Birth.8                              12\n",
      "Country.of.Birth.8                           12\n",
      "Home.Language.8                               2\n",
      "Dept.No..8                                   12\n",
      "Faculty.No..8                                12\n",
      "With.PHD.8                                    8\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.8       12\n",
      "Number.of.Successful.Grant.8                 12\n",
      "Number.of.Unsuccessful.Grant.8               12\n",
      "A..8                                         12\n",
      "A.8                                          12\n",
      "B.8                                          12\n",
      "C.8                                          12\n",
      "Person.ID.9                                   8\n",
      "Role.9                                       34\n",
      "Year.of.Birth.9                               8\n",
      "Country.of.Birth.9                            8\n",
      "Home.Language.9                               2\n",
      "Dept.No..9                                    8\n",
      "Faculty.No..9                                 8\n",
      "With.PHD.9                                    7\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.9        8\n",
      "Number.of.Successful.Grant.9                  8\n",
      "Number.of.Unsuccessful.Grant.9                8\n",
      "A..9                                          8\n",
      "A.9                                           8\n",
      "B.9                                           8\n",
      "C.9                                           8\n",
      "Person.ID.10                                  6\n",
      "Role.10                                      27\n",
      "Year.of.Birth.10                              6\n",
      "Country.of.Birth.10                           6\n",
      "Home.Language.10                              3\n",
      "Dept.No..10                                   6\n",
      "Faculty.No..10                                6\n",
      "With.PHD.10                                   5\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.10       6\n",
      "Number.of.Successful.Grant.10                 6\n",
      "Number.of.Unsuccessful.Grant.10               6\n",
      "A..10                                         6\n",
      "A.10                                          6\n",
      "B.10                                          6\n",
      "C.10                                          6\n",
      "Person.ID.11                                  5\n",
      "Role.11                                      19\n",
      "Year.of.Birth.11                              5\n",
      "Country.of.Birth.11                           5\n",
      "Home.Language.11                              0\n",
      "Dept.No..11                                   5\n",
      "Faculty.No..11                                0\n",
      "With.PHD.11                                   3\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.11       5\n",
      "Number.of.Successful.Grant.11                 5\n",
      "Number.of.Unsuccessful.Grant.11               5\n",
      "A..11                                         5\n",
      "A.11                                          5\n",
      "B.11                                          5\n",
      "C.11                                          5\n",
      "Person.ID.12                                  5\n",
      "Role.12                                      14\n",
      "Year.of.Birth.12                              5\n",
      "Country.of.Birth.12                           5\n",
      "Home.Language.12                              0\n",
      "Dept.No..12                                   5\n",
      "Faculty.No..12                                5\n",
      "With.PHD.12                                   5\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.12       5\n",
      "Number.of.Successful.Grant.12                 5\n",
      "Number.of.Unsuccessful.Grant.12               5\n",
      "A..12                                         5\n",
      "A.12                                          5\n",
      "B.12                                          5\n",
      "C.12                                          5\n",
      "Person.ID.13                                  3\n",
      "Role.13                                      11\n",
      "Year.of.Birth.13                              3\n",
      "Country.of.Birth.13                           3\n",
      "Home.Language.13                              0\n",
      "Dept.No..13                                   3\n",
      "Faculty.No..13                                3\n",
      "With.PHD.13                                   3\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.13       3\n",
      "Number.of.Successful.Grant.13                 3\n",
      "Number.of.Unsuccessful.Grant.13               3\n",
      "A..13                                         3\n",
      "A.13                                          3\n",
      "B.13                                          3\n",
      "C.13                                          3\n",
      "Person.ID.14                                  0\n",
      "Role.14                                       0\n",
      "Year.of.Birth.14                              3\n",
      "Country.of.Birth.14                           3\n",
      "Home.Language.14                              0\n",
      "Dept.No..14                                   0\n",
      "Faculty.No..14                                0\n",
      "With.PHD.14                                   0\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.14       0\n",
      "Number.of.Successful.Grant.14                 0\n",
      "Number.of.Unsuccessful.Grant.14               0\n",
      "A..14                                         0\n",
      "A.14                                          0\n",
      "B.14                                          0\n",
      "C.14                                          0\n",
      "Person.ID.15                                  2\n",
      "Role.15                                       8\n",
      "Year.of.Birth.15                              2\n",
      "Country.of.Birth.15                           2\n",
      "Home.Language.15                              0\n",
      "Dept.No..15                                   2\n",
      "Faculty.No..15                                2\n",
      "With.PHD.15                                   0\n",
      "No..of.Years.in.Uni.at.Time.of.Grant.15       2\n",
      "Number.of.Successful.Grant.15                 2\n",
      "Number.of.Unsuccessful.Grant.15               2\n",
      "A..15                                         2\n",
      "A.15                                          2\n",
      "B.15                                          2\n",
      "C.15                                          0\n",
      "Unnamed: 251                                  0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(data.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:09.652703",
     "start_time": "2017-02-14T14:32:09.647577"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['CHIEF_INVESTIGATOR', 'PRINCIPAL_SUPERVISOR',\n",
       "       'DELEGATED_RESEARCHER', 'EXT_CHIEF_INVESTIGATOR', nan,\n",
       "       'STUD_CHIEF_INVESTIGATOR', 'HONVISIT', 'EXTERNAL_ADVISOR'], dtype=object)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Role.1\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:09.988330",
     "start_time": "2017-02-14T14:32:09.983810"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Other', nan, 'English'], dtype=object)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Home.Language.1\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:10.231603",
     "start_time": "2017-02-14T14:32:10.226155"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, 'Yes '], dtype=object)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"With.PHD.3\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:10.420860",
     "start_time": "2017-02-14T14:32:10.416153"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Asia Pacific', 'Australia', 'Western Europe', nan, 'North America',\n",
       "       'Great Britain', 'New Zealand', 'The Americas', 'Eastern Europe',\n",
       "       'South Africa', 'Middle East and Africa'], dtype=object)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Country.of.Birth.1\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:10.595545",
     "start_time": "2017-02-14T14:32:10.591959"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t = \"Role.\"\n",
    "l = []\n",
    "for i in range(15):\n",
    "    l.append(t + str(i+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:10.756072",
     "start_time": "2017-02-14T14:32:10.751175"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "row_cols = data[l]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:32:11.223159",
     "start_time": "2017-02-14T14:32:11.207251"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                 Role.1              Role.2                  Role.3  \\\n",
       "0    CHIEF_INVESTIGATOR                 NaN                     NaN   \n",
       "1    CHIEF_INVESTIGATOR                 NaN                     NaN   \n",
       "2    CHIEF_INVESTIGATOR  CHIEF_INVESTIGATOR      CHIEF_INVESTIGATOR   \n",
       "3  PRINCIPAL_SUPERVISOR  CHIEF_INVESTIGATOR  EXT_CHIEF_INVESTIGATOR   \n",
       "4    CHIEF_INVESTIGATOR                 NaN                     NaN   \n",
       "\n",
       "                   Role.4                  Role.5                   Role.6  \\\n",
       "0                     NaN                     NaN                      NaN   \n",
       "1                     NaN                     NaN                      NaN   \n",
       "2    DELEGATED_RESEARCHER  EXT_CHIEF_INVESTIGATOR   EXT_CHIEF_INVESTIGATOR   \n",
       "3  EXT_CHIEF_INVESTIGATOR  EXT_CHIEF_INVESTIGATOR  STUD_CHIEF_INVESTIGATOR   \n",
       "4                     NaN                     NaN                      NaN   \n",
       "\n",
       "                   Role.7 Role.8 Role.9 Role.10 Role.11 Role.12 Role.13  \\\n",
       "0                     NaN    NaN    NaN     NaN     NaN     NaN     NaN   \n",
       "1                     NaN    NaN    NaN     NaN     NaN     NaN     NaN   \n",
       "2  EXT_CHIEF_INVESTIGATOR    NaN    NaN     NaN     NaN     NaN     NaN   \n",
       "3                     NaN    NaN    NaN     NaN     NaN     NaN     NaN   \n",
       "4                     NaN    NaN    NaN     NaN     NaN     NaN     NaN   \n",
       "\n",
       "   Role.14 Role.15  \n",
       "0      NaN     NaN  \n",
       "1      NaN     NaN  \n",
       "2      NaN     NaN  \n",
       "3      NaN     NaN  \n",
       "4      NaN     NaN  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row_cols.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-02-14T14:34:54.493878",
     "start_time": "2017-02-14T14:34:54.486680"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[26]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argwhere(data.columns == \"Person.ID.1\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}