notebooks/census/Zimbabwe Raw Data Full Simulated School Mining Manufacturing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 537 ms\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "\n",
    "from covid19_abm.dir_manager import get_data_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 662 µs\n"
     ]
    }
   ],
   "source": [
    "age_map = {\n",
    "    'less than 1 year': 0,\n",
    "    'less than 1 year ': 0,\n",
    "    '1 year': 1,\n",
    "    '1 year ': 1,\n",
    "    '2 years': 2,\n",
    "    '2 years ': 2,\n",
    "    '98': 98,\n",
    "    '98 ': 98,\n",
    "    'not reported/missing': None,\n",
    "    'not reported/missing ': None,\n",
    "}\n",
    "\n",
    "econ_stat_map = {\n",
    "    'Not working, inactive, not': 'Not working, inactive, not in universe',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1min 42s\n"
     ]
    }
   ],
   "source": [
    "full_individual_df = pd.read_stata(\n",
    "    get_data_dir('raw', 'census', 'ABM_Simulated_Pop_WardDistributed_UpdatedMay30_school_complete_060520.dta'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 3.5 s\n"
     ]
    }
   ],
   "source": [
    "full_individual_df['age'] = full_individual_df['age'].map(lambda x: age_map.get(x, x))\n",
    "full_individual_df['economic_status'] = full_individual_df['economic_status'].str.strip().map(lambda x: econ_stat_map.get(x, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serial</th>\n",
       "      <th>persons</th>\n",
       "      <th>district_id</th>\n",
       "      <th>new_district_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>teachers</th>\n",
       "      <th>nfams</th>\n",
       "      <th>isco88a</th>\n",
       "      <th>famunit</th>\n",
       "      <th>...</th>\n",
       "      <th>new_dist_pop_share</th>\n",
       "      <th>geo1_zw</th>\n",
       "      <th>geo1_zw2012</th>\n",
       "      <th>geo1_string</th>\n",
       "      <th>geo2_zw</th>\n",
       "      <th>geo2_zw2012</th>\n",
       "      <th>dhs_ipumsi_zw</th>\n",
       "      <th>province_id_shpfile</th>\n",
       "      <th>province_name_shpfile</th>\n",
       "      <th>district_name_shpfile</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>67786</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>domestic and related helpers, cleaners and lau...</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>64925</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>travel attendants and related workers</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3648785</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>female</td>\n",
       "      <td>Secondary education teaching professionals</td>\n",
       "      <td>4 families</td>\n",
       "      <td>secondary education teaching professionals</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>127965</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>103824</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>65</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    serial  persons district_id  new_district_id age     sex  \\\n",
       "0    67786        4          21                1  19  female   \n",
       "1    64925        3          21                1  49    male   \n",
       "2  3648785        8          21                1  26  female   \n",
       "3   127965        6          21                1  54  female   \n",
       "4   103824        5          21                1  65  female   \n",
       "\n",
       "                                     teachers       nfams  \\\n",
       "0                                           0  2 families   \n",
       "1                                           0  2 families   \n",
       "2  Secondary education teaching professionals  4 families   \n",
       "3                                           0  2 families   \n",
       "4                                           0    1 family   \n",
       "\n",
       "                                             isco88a  famunit  ...  \\\n",
       "0  domestic and related helpers, cleaners and lau...        2  ...   \n",
       "1              travel attendants and related workers        1  ...   \n",
       "2         secondary education teaching professionals        2  ...   \n",
       "3                              niu (not in universe)        1  ...   \n",
       "4                              niu (not in universe)        1  ...   \n",
       "\n",
       "  new_dist_pop_share   geo1_zw geo1_zw2012 geo1_string         geo2_zw  \\\n",
       "0                1.0  bulawayo    bulawayo           0  bulawayo urban   \n",
       "1                1.0  bulawayo    bulawayo           0  bulawayo urban   \n",
       "2                1.0  bulawayo    bulawayo           0  bulawayo urban   \n",
       "3                1.0  bulawayo    bulawayo           0  bulawayo urban   \n",
       "4                1.0  bulawayo    bulawayo           0  bulawayo urban   \n",
       "\n",
       "      geo2_zw2012 dhs_ipumsi_zw province_id_shpfile province_name_shpfile  \\\n",
       "0  bulawayo urban      bulawayo                   1              Bulawayo   \n",
       "1  bulawayo urban      bulawayo                   1              Bulawayo   \n",
       "2  bulawayo urban      bulawayo                   1              Bulawayo   \n",
       "3  bulawayo urban      bulawayo                   1              Bulawayo   \n",
       "4  bulawayo urban      bulawayo                   1              Bulawayo   \n",
       "\n",
       "  district_name_shpfile  \n",
       "0              Bulawayo  \n",
       "1              Bulawayo  \n",
       "2              Bulawayo  \n",
       "3              Bulawayo  \n",
       "4              Bulawayo  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 29 ms\n"
     ]
    }
   ],
   "source": [
    "full_individual_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Not working, inactive, not in universe    5059625\n",
       "In School                                 2828795\n",
       "Homemakers/Housework                      1267806\n",
       "Office workers                             465489\n",
       "Service Workers                           2140208\n",
       "Agriculture Workers                       2309064\n",
       "Indusrtry Workers                          121116\n",
       "In the army                                 41494\n",
       "Disabled and not working                   169548\n",
       "Name: economic_status, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 802 ms\n"
     ]
    }
   ],
   "source": [
    "l = '''Not working, inactive, not in universe\n",
    "In School\n",
    "Homemakers/Housework\n",
    "Office workers\n",
    "Service Workers\n",
    "Agriculture Workers\n",
    "Indusrtry Workers\n",
    "In the army\n",
    "Disabled and not working'''\n",
    "l = l.split('\\n')\n",
    "full_individual_df['economic_status'].value_counts()[l]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fill unknown age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 385 µs\n"
     ]
    }
   ],
   "source": [
    "individual_df = full_individual_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 11.6 s\n"
     ]
    }
   ],
   "source": [
    "# missing_val = 'not reported/missing'\n",
    "age_cols = ['geo1_zw2012', 'urban', 'persons', 'sex', 'marst', 'citizen', 'race', 'disabled', 'economic_status']\n",
    "X = pd.get_dummies(individual_df[age_cols], drop_first=True)\n",
    "\n",
    "X_train = X[individual_df['age'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 57.4 s\n"
     ]
    }
   ],
   "source": [
    "gb = GradientBoostingRegressor(n_estimators=100, max_depth=3, random_state=1029)\n",
    "# rf = RandomForestRegressor(n_estimators=100, max_depth=3, random_state=1029, n_jobs=50)\n",
    "\n",
    "frac = 0.05\n",
    "X_tr = X_train.sample(frac=frac, random_state=1029)\n",
    "gb.fit(X_tr, individual_df.loc[X_tr.index, 'age'])\n",
    "\n",
    "missing_age = X[individual_df['age'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fa9dffc3d90>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAXqUlEQVR4nO3df7DddX3n8edrScUIlQTRu2yS2eA2Q0VSLdwBrDudC7QQ0DH8oTswTAludjLTxRa72dGwbpetyixOS6nOVnaykgquQ6TULhlAMYPccbojyE8NiCwRMhBBopuARqx67Xv/OJ+4Z8PN/XHuzT1fy/Mxc+ac7/v7+X6/r/M99+Z9v9/zPSepKiRJr2z/ZNgBJEnDZzOQJNkMJEk2A0kSNgNJErBo2AEGddxxx9XKlSuHHeMXfvSjH3HUUUcNO8akupwNup2vy9mg2/m6nA26ne9wZnvggQe+X1Wvf9mMqvqlvJ166qnVJXffffewIxxSl7NVdTtfl7NVdTtfl7NVdTvf4cwG3F+T/JvqaSJJks1AkmQzkCRhM5AkYTOQJGEzkCRhM5AkYTOQJGEzkCQxg6+jSLIFeCewp6pOPmjevwf+FHh9VX0/SYCPA+cDLwGXVtWDbew64D+2RT9aVTe0+qnAp4HFwB3A5e1TcofNyk23z/s6N66e4NJp1rvr6nfM+3YlaT7M5Mjg08Cag4tJVgC/CzzdVz4PWNVuG4Dr2thjgSuB04HTgCuTLG3LXNfGHljuZduSJB1e0zaDqvoKsHeSWdcCHwD6/4pfC9zYvgLjHmBJkuOBc4HtVbW3qvYB24E1bd5rq+qr7WjgRuCCuT0lSdJsDfStpUneBXynqr7eOzP0C8uAZ/qmd7faVPXdk9QPtd0N9I4iGBkZYXx8fJD4bFw9MdByUxlZPP16B807V/v37x/atmeiy/m6nA26na/L2aDb+YaRbdbNIMlrgA8B50w2e5JaDVCfVFVtBjYDjI6O1tjY2HRxJzXduf1BbFw9wTU7pt6duy4em/ftzsT4+DiD7quF0OV8Xc4G3c7X5WzQ7XzDyDbI1UT/AjgB+HqSXcBy4MEk/5TeX/Yr+sYuB56dpr58krokaQHNuhlU1Y6qekNVrayqlfT+QT+lqr4LbAMuSc8ZwItV9RxwJ3BOkqXtjeNzgDvbvB8mOaNdiXQJcOs8PTdJ0gxN2wyS3AR8FTgxye4k66cYfgfwJLAT+O/AvwWoqr3AR4D72u3DrQbw+8Cn2jLfBr4w2FORJA1q2vcMquqiaeav7HtcwGWHGLcF2DJJ/X7g5JcvIUlaKH4CWZJkM5Ak2QwkSdgMJEnYDCRJ2AwkSdgMJEnYDCRJ2AwkSdgMJEnYDCRJ2AwkSdgMJEnYDCRJ2AwkSdgMJEnYDCRJ2AwkSdgMJEnYDCRJzKAZJNmSZE+SR/pqf5rkW0m+keRvkyzpm3dFkp1JHk9ybl99TavtTLKpr35CknuTPJHkc0leNZ9PUJI0vZkcGXwaWHNQbTtwclX9BvC/gSsAkpwEXAi8uS3zySRHJDkC+EvgPOAk4KI2FuBjwLVVtQrYB6yf0zOSJM3atM2gqr4C7D2o9qWqmmiT9wDL2+O1wNaq+klVPQXsBE5rt51V9WRV/RTYCqxNEuAs4Ja2/A3ABXN8TpKkWVo0D+v418Dn2uNl9JrDAbtbDeCZg+qnA68DXuhrLP3jXybJBmADwMjICOPj4wMF3rh6YvpBszSyePr1Dpp3rvbv3z+0bc9El/N1ORt0O1+Xs0G38w0j25yaQZIPARPAZw+UJhlWTH4EUlOMn1RVbQY2A4yOjtbY2Nhs4v7CpZtuH2i5qWxcPcE1O6benbsuHpv37c7E+Pg4g+6rhdDlfF3OBt3O1+Vs0O18w8g2cDNIsg54J3B2VR34B3w3sKJv2HLg2fZ4svr3gSVJFrWjg/7xkqQFMtClpUnWAB8E3lVVL/XN2gZcmOTIJCcAq4CvAfcBq9qVQ6+i9ybzttZE7gbe3ZZfB9w62FORJA1qJpeW3gR8FTgxye4k64H/CvwqsD3Jw0n+G0BVPQrcDHwT+CJwWVX9vP3V/z7gTuAx4OY2FnpN5d8l2UnvPYTr5/UZSpKmNe1poqq6aJLyIf/BrqqrgKsmqd8B3DFJ/Ul6VxtJkobETyBLkmwGkiSbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJIkZNIMkW5LsSfJIX+3YJNuTPNHul7Z6knwiyc4k30hySt8y69r4J5Ks66ufmmRHW+YTSTLfT1KSNLWZHBl8GlhzUG0TcFdVrQLuatMA5wGr2m0DcB30mgdwJXA6cBpw5YEG0sZs6Fvu4G1Jkg6zaZtBVX0F2HtQeS1wQ3t8A3BBX/3G6rkHWJLkeOBcYHtV7a2qfcB2YE2b99qq+mpVFXBj37okSQtk0YDLjVTVcwBV9VySN7T6MuCZvnG7W22q+u5J6pNKsoHeUQQjIyOMj48PFH7j6omBlpvKyOLp1zto3rnav3//0LY9E13O1+Vs0O18Xc4G3c43jGyDNoNDmex8fw1Qn1RVbQY2A4yOjtbY2NgAEeHSTbcPtNxUNq6e4JodU+/OXRePzft2Z2J8fJxB99VC6HK+LmeDbufrcjbodr5hZBv0aqLn2yke2v2eVt8NrOgbtxx4dpr68knqkqQFNGgz2AYcuCJoHXBrX/2SdlXRGcCL7XTSncA5SZa2N47PAe5s836Y5Ix2FdElfeuSJC2QaU8TJbkJGAOOS7Kb3lVBVwM3J1kPPA28pw2/Azgf2Am8BLwXoKr2JvkIcF8b9+GqOvCm9O/Tu2JpMfCFdpMkLaBpm0FVXXSIWWdPMraAyw6xni3Alknq9wMnT5dDknT4+AlkSZLNQJJkM5AkYTOQJGEzkCRhM5AkYTOQJGEzkCRhM5AkYTOQJGEzkCRhM5AkYTOQJGEzkCRhM5AkYTOQJGEzkCRhM5AkYTOQJGEzkCQxx2aQ5I+SPJrkkSQ3JXl1khOS3JvkiSSfS/KqNvbINr2zzV/Zt54rWv3xJOfO7SlJkmZr4GaQZBnwh8BoVZ0MHAFcCHwMuLaqVgH7gPVtkfXAvqr6NeDaNo4kJ7Xl3gysAT6Z5IhBc0mSZm+up4kWAYuTLAJeAzwHnAXc0ubfAFzQHq9t07T5ZydJq2+tqp9U1VPATuC0OeaSJM1CqmrwhZPLgauAHwNfAi4H7ml//ZNkBfCFqjo5ySPAmqra3eZ9Gzgd+M9tmf/R6te3ZW6ZZHsbgA0AIyMjp27dunWg3Du+8+JAy01lZDE8/+Opx6xedsy8b3cm9u/fz9FHHz2Ubc9El/N1ORt0O1+Xs0G38x3ObGeeeeYDVTV6cH3RoCtMspTeX/UnAC8Afw2cN8nQA90mh5h3qPrLi1Wbgc0Ao6OjNTY2NrvQzaWbbh9oualsXD3BNTum3p27Lh6b9+3OxPj4OIPuq4XQ5XxdzgbdztflbNDtfMPINpfTRL8DPFVV36uqnwGfB34LWNJOGwEsB55tj3cDKwDa/GOAvf31SZaRJC2AuTSDp4Ezkrymnfs/G/gmcDfw7jZmHXBre7ytTdPmf7l656i2ARe2q41OAFYBX5tDLknSLA18mqiq7k1yC/AgMAE8RO8Uzu3A1iQfbbXr2yLXA59JspPeEcGFbT2PJrmZXiOZAC6rqp8PmkuSNHsDNwOAqroSuPKg8pNMcjVQVf098J5DrOcqem9ES5KGwE8gS5JsBpIkm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSQJm4EkCZuBJAmbgSSJOTaDJEuS3JLkW0keS/K2JMcm2Z7kiXa/tI1Nkk8k2ZnkG0lO6VvPujb+iSTr5vqkJEmzM9cjg48DX6yqXwfeAjwGbALuqqpVwF1tGuA8YFW7bQCuA0hyLHAlcDpwGnDlgQYiSVoYAzeDJK8Ffhu4HqCqflpVLwBrgRvasBuAC9rjtcCN1XMPsCTJ8cC5wPaq2ltV+4DtwJpBc0mSZm8uRwZvBL4H/FWSh5J8KslRwEhVPQfQ7t/Qxi8DnulbfnerHaouSVogqarBFkxGgXuAt1fVvUk+DvwA+IOqWtI3bl9VLU1yO/BfqurvWv0u4APAWcCRVfXRVv9j4KWqumaSbW6gd4qJkZGRU7du3TpQ9h3feXGg5aYyshie//HUY1YvO2betzsT+/fv5+ijjx7Ktmeiy/m6nA26na/L2aDb+Q5ntjPPPPOBqho9uL5oDuvcDeyuqnvb9C303h94PsnxVfVcOw20p2/8ir7llwPPtvrYQfXxyTZYVZuBzQCjo6M1NjY22bBpXbrp9oGWm8rG1RNcs2Pq3bnr4rF53+5MjI+PM+i+WghdztflbNDtfF3OBt3ON4xsA58mqqrvAs8kObGVzga+CWwDDlwRtA64tT3eBlzSrio6A3ixnUa6EzgnydL2xvE5rSZJWiBzOTIA+APgs0leBTwJvJdeg7k5yXrgaeA9bewdwPnATuClNpaq2pvkI8B9bdyHq2rvHHNJkmZhTs2gqh4GXnbuid5RwsFjC7jsEOvZAmyZSxZJ0uD8BLIkyWYgSbIZSJKwGUiSsBlIkrAZSJKwGUiSsBlIkrAZSJKwGUiSsBlIkrAZSJKwGUiSsBlIkrAZSJKwGUiSsBlIkpj7f3spddbKTbfPeR0bV09w6SzXs+vqd8x5u9JC88hAkmQzkCTNQzNIckSSh5Lc1qZPSHJvkieSfC7Jq1r9yDa9s81f2beOK1r98STnzjWTJGl25uPI4HLgsb7pjwHXVtUqYB+wvtXXA/uq6teAa9s4kpwEXAi8GVgDfDLJEfOQS5I0Q3NqBkmWA+8APtWmA5wF3NKG3ABc0B6vbdO0+We38WuBrVX1k6p6CtgJnDaXXJKk2ZnrkcFfAB8A/qFNvw54oaom2vRuYFl7vAx4BqDNf7GN/0V9kmUkSQtg4EtLk7wT2FNVDyQZO1CeZGhNM2+qZQ7e5gZgA8DIyAjj4+OzifwLG1dPTD9olkYWT7/eQfPO1f79+4e27Zk4XPnm43Weyet6sIXc111+bbucDbqdbxjZ5vI5g7cD70pyPvBq4LX0jhSWJFnU/vpfDjzbxu8GVgC7kywCjgH29tUP6F/m/1NVm4HNAKOjozU2NjZQ8NleNz4TG1dPcM2OqXfnrovH5n27MzE+Ps6g+2ohHK588/E6z+R1PdhCvs5dfm27nA26nW8Y2QY+TVRVV1TV8qpaSe8N4C9X1cXA3cC727B1wK3t8bY2TZv/5aqqVr+wXW10ArAK+NqguSRJs3c4PoH8QWBrko8CDwHXt/r1wGeS7KR3RHAhQFU9muRm4JvABHBZVf38MOSSJB3CvDSDqhoHxtvjJ5nkaqCq+nvgPYdY/irgqvnIIkmaPT+BLEmyGUiSbAaSJGwGkiRsBpIkbAaSJGwGkiRsBpIk/D+QF9R8/J+8g9i4eoKxoWxZ0i8LjwwkSTYDSZLNQJKEzUCShM1AkoTNQJKEzUCShM1AkoTNQJKEzUCShF9HocNsJl/BsXH1BJcO6as6JPV4ZCBJGrwZJFmR5O4kjyV5NMnlrX5sku1Jnmj3S1s9ST6RZGeSbyQ5pW9d69r4J5Ksm/vTkiTNxlyODCaAjVX1JuAM4LIkJwGbgLuqahVwV5sGOA9Y1W4bgOug1zyAK4HTgdOAKw80EEnSwhi4GVTVc1X1YHv8Q+AxYBmwFrihDbsBuKA9XgvcWD33AEuSHA+cC2yvqr1VtQ/YDqwZNJckafZSVXNfSbIS+ApwMvB0VS3pm7evqpYmuQ24uqr+rtXvAj4IjAGvrqqPtvofAz+uqj+bZDsb6B1VMDIycurWrVsHyrvjOy8OtNxURhbD8z+e99XOi5HF8IZjjxnKtmeyr7u+72abbfWyhdvX+/fv5+ijj16w7c1Gl7NBt/MdzmxnnnnmA1U1enB9zlcTJTka+Bvg/VX1gySHHDpJraaov7xYtRnYDDA6OlpjY2OzzgsclitXNq6e4Jod3bw4a+PqCf7VgPtqrmayr7u+72abbdfFY4cnzCTGx8cZ9PfgcOtyNuh2vmFkm9PVREl+hV4j+GxVfb6Vn2+nf2j3e1p9N7Cib/HlwLNT1CVJC2QuVxMFuB54rKr+vG/WNuDAFUHrgFv76pe0q4rOAF6squeAO4Fzkixtbxyf02qSpAUyl2PztwO/B+xI8nCr/QfgauDmJOuBp4H3tHl3AOcDO4GXgPcCVNXeJB8B7mvjPlxVe+eQS5I0SwM3g/ZG8KHeIDh7kvEFXHaIdW0BtgyaRZI0N34CWZJkM5Ak2QwkSdgMJEn4FdbSPyoz+crww2HX1e8Yynbn8nzn+tXpw3rOh4vN4BViWP9ISPrl4GkiSZLNQJLkaSJJGsjhPPU61fsZh+u9Co8MJEkeGUiau8n+Sp7r1TpaWB4ZSJI8MpDm20Jexutf35ovHhlIkmwGkiSbgSQJm4EkCZuBJAmbgSQJm4EkiQ41gyRrkjyeZGeSTcPOI0mvJJ1oBkmOAP4SOA84CbgoyUnDTSVJrxydaAbAacDOqnqyqn4KbAXWDjmTJL1ipKqGnYEk7wbWVNW/adO/B5xeVe87aNwGYEObPBF4fEGDTu044PvDDnEIXc4G3c7X5WzQ7Xxdzgbdznc4s/3zqnr9wcWufDdRJqm9rEtV1WZg8+GPM3tJ7q+q0WHnmEyXs0G383U5G3Q7X5ezQbfzDSNbV04T7QZW9E0vB54dUhZJesXpSjO4D1iV5IQkrwIuBLYNOZMkvWJ04jRRVU0keR9wJ3AEsKWqHh1yrNnq5OmrpsvZoNv5upwNup2vy9mg2/kWPFsn3kCWJA1XV04TSZKGyGYgSbIZDCLJliR7kjzSVzs2yfYkT7T7pUPKtiLJ3UkeS/Joksu7ki/Jq5N8LcnXW7Y/afUTktzbsn2uXUQwNEmOSPJQktu6lC/JriQ7kjyc5P5WG/rr2pdvSZJbknyr/fy9rQv5kpzY9tmB2w+SvL8L2foy/lH7nXgkyU3td2VBf+5sBoP5NLDmoNom4K6qWgXc1aaHYQLYWFVvAs4ALmtf7dGFfD8BzqqqtwBvBdYkOQP4GHBty7YPWD+EbP0uBx7rm+5SvjOr6q1916B34XU94OPAF6vq14G30NuHQ89XVY+3ffZW4FTgJeBvu5ANIMky4A+B0ao6md5FNBey0D93VeVtgBuwEnikb/px4Pj2+Hjg8WFnbFluBX63a/mA1wAPAqfT+6TlolZ/G3DnEHMtp/cPw1nAbfQ+ENmJfMAu4LiDap14XYHXAk/RLkrpWr6+POcA/6tL2YBlwDPAsfSu8LwNOHehf+48Mpg/I1X1HEC7f8OQ85BkJfCbwL10JF87BfMwsAfYDnwbeKGqJtqQ3fR+OYblL4APAP/Qpl9Hd/IV8KUkD7SvZoGOvK7AG4HvAX/VTrF9KslRHcp3wIXATe1xJ7JV1XeAPwOeBp4DXgQeYIF/7mwG/0glORr4G+D9VfWDYec5oKp+Xr3D9eX0vqDwTZMNW9hUPUneCeypqgf6y5MMHdb12G+vqlPofbvvZUl+e0g5JrMIOAW4rqp+E/gRwz1l9TLtnPu7gL8edpZ+7b2KtcAJwD8DjqL3Gh/ssP7c2Qzmz/NJjgdo93uGFSTJr9BrBJ+tqs93LR9AVb0AjNN7X2NJkgMfgBzmV5G8HXhXkl30vjn3LHpHCp3IV1XPtvs99M55n0Z3XtfdwO6qurdN30KvOXQlH/T+gX2wqp5v013J9jvAU1X1var6GfB54LdY4J87m8H82Qasa4/X0TtXv+CSBLgeeKyq/rxv1tDzJXl9kiXt8WJ6vwSPAXcD7x5mNoCquqKqllfVSnqnE75cVRd3IV+So5L86oHH9M59P0IHXleAqvou8EySE1vpbOCbdCRfcxH/7xQRdCfb08AZSV7Tfn8P7LuF/bkb5ps5v6w3ej9QzwE/o/cX0Xp655bvAp5o98cOKdu/pHc4+Q3g4XY7vwv5gN8AHmrZHgH+U6u/EfgasJPeIfyRHXiNx4DbupKvZfh6uz0KfKjVh/669mV8K3B/e33/J7C0K/noXbDwf4Bj+mqdyNay/AnwrfZ78RngyIX+ufPrKCRJniaSJNkMJEnYDCRJ2AwkSdgMJEnYDCRJ2AwkScD/BV10rkVwzNjxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 264 ms\n"
     ]
    }
   ],
   "source": [
    "pd.Series(gb.predict(missing_age)).hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(718623, 33)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.36 ms\n"
     ]
    }
   ],
   "source": [
    "X_tr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serial</th>\n",
       "      <th>persons</th>\n",
       "      <th>district_id</th>\n",
       "      <th>new_district_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>teachers</th>\n",
       "      <th>nfams</th>\n",
       "      <th>isco88a</th>\n",
       "      <th>famunit</th>\n",
       "      <th>...</th>\n",
       "      <th>new_dist_pop_share</th>\n",
       "      <th>geo1_zw</th>\n",
       "      <th>geo1_zw2012</th>\n",
       "      <th>geo1_string</th>\n",
       "      <th>geo2_zw</th>\n",
       "      <th>geo2_zw2012</th>\n",
       "      <th>dhs_ipumsi_zw</th>\n",
       "      <th>province_id_shpfile</th>\n",
       "      <th>province_name_shpfile</th>\n",
       "      <th>district_name_shpfile</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>438</td>\n",
       "      <td>64898</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>motor-vehicle drivers</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3978</td>\n",
       "      <td>141744</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6720</td>\n",
       "      <td>55301</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7963</td>\n",
       "      <td>143945</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>8 families</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9596</td>\n",
       "      <td>99018</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>domestic and related helpers, cleaners and lau...</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 59 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      serial  persons district_id  new_district_id age     sex teachers  \\\n",
       "438    64898        3          21                1  37    male        0   \n",
       "3978  141744        8          21                1   9    male        0   \n",
       "6720   55301        3          21                1  19    male        0   \n",
       "7963  143945        8          21                1   7    male        0   \n",
       "9596   99018        5          21                1  20  female        0   \n",
       "\n",
       "           nfams                                            isco88a  famunit  \\\n",
       "438     1 family                              motor-vehicle drivers        1   \n",
       "3978    1 family                              niu (not in universe)        1   \n",
       "6720    1 family                              niu (not in universe)        1   \n",
       "7963  8 families                              niu (not in universe)        8   \n",
       "9596  2 families  domestic and related helpers, cleaners and lau...        2   \n",
       "\n",
       "      ... new_dist_pop_share   geo1_zw geo1_zw2012 geo1_string  \\\n",
       "438   ...                1.0  bulawayo    bulawayo           0   \n",
       "3978  ...                1.0  bulawayo    bulawayo           0   \n",
       "6720  ...                1.0  bulawayo    bulawayo           0   \n",
       "7963  ...                1.0  bulawayo    bulawayo           0   \n",
       "9596  ...                1.0  bulawayo    bulawayo           0   \n",
       "\n",
       "             geo2_zw     geo2_zw2012 dhs_ipumsi_zw province_id_shpfile  \\\n",
       "438   bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "3978  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "6720  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "7963  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "9596  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "\n",
       "     province_name_shpfile district_name_shpfile  \n",
       "438               Bulawayo              Bulawayo  \n",
       "3978              Bulawayo              Bulawayo  \n",
       "6720              Bulawayo              Bulawayo  \n",
       "7963              Bulawayo              Bulawayo  \n",
       "9596              Bulawayo              Bulawayo  \n",
       "\n",
       "[5 rows x 59 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 940 ms\n"
     ]
    }
   ],
   "source": [
    "individual_df.loc[missing_age.index, 'age'] = gb.predict(missing_age).astype(int)\n",
    "individual_df.loc[missing_age.index].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Not working, inactive, not in universe    5059625\n",
       "In School                                 2828795\n",
       "Agriculture Workers                       2309064\n",
       "Service Workers                           2140208\n",
       "Homemakers/Housework                      1267806\n",
       "Office workers                             465489\n",
       "Disabled and not working                   169548\n",
       "Indusrtry Workers                          121116\n",
       "In the army                                 41494\n",
       "Name: economic_status, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.92 s\n"
     ]
    }
   ],
   "source": [
    "individual_df.economic_status.astype(str).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create `district_id`, `household_id` and `economic_activity_location_id`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 567 µs\n"
     ]
    }
   ],
   "source": [
    "relevant_cols = [\n",
    "    'person_id', 'age', 'sex',\n",
    "    'household_id', 'district_id',\n",
    "    'economic_status', 'economic_activity_location_id',\n",
    "    # 'household', 'district'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 4.43 ms\n"
     ]
    }
   ],
   "source": [
    "old_new_districts = pd.read_csv(\n",
    "    get_data_dir('raw', 'district_relation.csv', index_col=0))\n",
    "old_new_districts.index = old_new_districts['ADMIN_NAME'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 47.2 ms\n"
     ]
    }
   ],
   "source": [
    "individual_df['serial_expanded'] = individual_df['serial']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 5.52 s\n"
     ]
    }
   ],
   "source": [
    "individual_df['household_id'] = individual_df['serial_expanded'].map(lambda x: f'h_{x}')\n",
    "individual_df['old_district_id'] = individual_df['geo2_zw2012'].map(lambda x: f'd_{old_new_districts[\"DIST2012\"][x]}')                                          \n",
    "individual_df['new_district_id'] = individual_df['geo2_zw2012'].map(lambda x: f'd_{old_new_districts[\"NEW_DIST_ID_2\"][x]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 9.29 s\n"
     ]
    }
   ],
   "source": [
    "expanded_individual_df = individual_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serial</th>\n",
       "      <th>persons</th>\n",
       "      <th>district_id</th>\n",
       "      <th>new_district_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>teachers</th>\n",
       "      <th>nfams</th>\n",
       "      <th>isco88a</th>\n",
       "      <th>famunit</th>\n",
       "      <th>...</th>\n",
       "      <th>geo2_zw</th>\n",
       "      <th>geo2_zw2012</th>\n",
       "      <th>dhs_ipumsi_zw</th>\n",
       "      <th>province_id_shpfile</th>\n",
       "      <th>province_name_shpfile</th>\n",
       "      <th>district_name_shpfile</th>\n",
       "      <th>serial_expanded</th>\n",
       "      <th>household_id</th>\n",
       "      <th>old_district_id</th>\n",
       "      <th>new_district_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>67786</td>\n",
       "      <td>4</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>domestic and related helpers, cleaners and lau...</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>67786</td>\n",
       "      <td>h_67786</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>64925</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>travel attendants and related workers</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>64925</td>\n",
       "      <td>h_64925</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3648785</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>female</td>\n",
       "      <td>Secondary education teaching professionals</td>\n",
       "      <td>4 families</td>\n",
       "      <td>secondary education teaching professionals</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>3648785</td>\n",
       "      <td>h_3648785</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>127965</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>54</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>2 families</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>127965</td>\n",
       "      <td>h_127965</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>103824</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>65</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo urban</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>103824</td>\n",
       "      <td>h_103824</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 63 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    serial  persons district_id  new_district_id age     sex  \\\n",
       "0    67786        4          21                1  19  female   \n",
       "1    64925        3          21                1  49    male   \n",
       "2  3648785        8          21                1  26  female   \n",
       "3   127965        6          21                1  54  female   \n",
       "4   103824        5          21                1  65  female   \n",
       "\n",
       "                                     teachers       nfams  \\\n",
       "0                                           0  2 families   \n",
       "1                                           0  2 families   \n",
       "2  Secondary education teaching professionals  4 families   \n",
       "3                                           0  2 families   \n",
       "4                                           0    1 family   \n",
       "\n",
       "                                             isco88a  famunit  ...  \\\n",
       "0  domestic and related helpers, cleaners and lau...        2  ...   \n",
       "1              travel attendants and related workers        1  ...   \n",
       "2         secondary education teaching professionals        2  ...   \n",
       "3                              niu (not in universe)        1  ...   \n",
       "4                              niu (not in universe)        1  ...   \n",
       "\n",
       "          geo2_zw     geo2_zw2012 dhs_ipumsi_zw province_id_shpfile  \\\n",
       "0  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "1  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "2  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "3  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "4  bulawayo urban  bulawayo urban      bulawayo                   1   \n",
       "\n",
       "  province_name_shpfile district_name_shpfile serial_expanded household_id  \\\n",
       "0              Bulawayo              Bulawayo           67786      h_67786   \n",
       "1              Bulawayo              Bulawayo           64925      h_64925   \n",
       "2              Bulawayo              Bulawayo         3648785    h_3648785   \n",
       "3              Bulawayo              Bulawayo          127965     h_127965   \n",
       "4              Bulawayo              Bulawayo          103824     h_103824   \n",
       "\n",
       "  old_district_id new_district_id  \n",
       "0        w_21         w_1  \n",
       "1        w_21         w_1  \n",
       "2        w_21         w_1  \n",
       "3        w_21         w_1  \n",
       "4        w_21         w_1  \n",
       "\n",
       "[5 rows x 63 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 24.3 ms\n"
     ]
    }
   ],
   "source": [
    "expanded_individual_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 139 ms\n"
     ]
    }
   ],
   "source": [
    "expanded_individual_df['school_goers'] = 1 * (expanded_individual_df['school_goers'] != 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['serial', 'persons', 'district_id', 'new_district_id', 'age', 'sex',\n",
       "       'teachers', 'nfams', 'isco88a', 'famunit', 'urban', 'economic_status',\n",
       "       'gq', 'marst', 'citizen', 'race', 'disabled', 'disblnd', 'disdeaf',\n",
       "       'dismntl', 'famsize', 'age2', 'edattain', 'edattaind', 'yrschool',\n",
       "       'educzw', 'empstat', 'empstatd', 'labforce', 'occisco', 'occ',\n",
       "       'manufacturing_workers', 'mining_workers', 'mining_districts', 'pid',\n",
       "       'weight', 'district', 'school_goers', 'school_goers_detailed', 'pernum',\n",
       "       'geolev1', 'Province', 'Province_str', 'ProvincePop2012',\n",
       "       'ProvincePop2019', 'geolev2', 'new_district_id_st', 'NewDistPop2012',\n",
       "       'NewDistPop2019', 'new_dist_pop_share', 'geo1_zw', 'geo1_zw2012',\n",
       "       'geo1_string', 'geo2_zw', 'geo2_zw2012', 'dhs_ipumsi_zw',\n",
       "       'province_id_shpfile', 'province_name_shpfile', 'district_name_shpfile',\n",
       "       'serial_expanded', 'household_id', 'old_district_id', 'new_district_id'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.08 ms\n"
     ]
    }
   ],
   "source": [
    "expanded_individual_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.739394381540564"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.66 s\n"
     ]
    }
   ],
   "source": [
    "expanded_individual_df.groupby('serial')['sex'].count().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.739394381540564"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 505 ms\n"
     ]
    }
   ],
   "source": [
    "expanded_individual_df.shape[0] / expanded_individual_df['serial'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2923907"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 28.4 ms\n"
     ]
    }
   ],
   "source": [
    "(expanded_individual_df['school_goers'] == 1).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2518537"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 22.9 ms\n"
     ]
    }
   ],
   "source": [
    "(expanded_individual_df['school_goers'] == 1).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 609 µs\n"
     ]
    }
   ],
   "source": [
    "# relevant_cols = [\n",
    "#     'person_id', 'age', 'sex',\n",
    "#     'household_id', 'district_id',\n",
    "#     'economic_status', 'economic_activity_location_id',\n",
    "#     'school_id_district', \n",
    "#     'manufacturing_workers',\n",
    "#     'mining_district_id'\n",
    "#     # 'household', 'district'\n",
    "# ]\n",
    "\n",
    "relevant_cols = [\n",
    "    'person_id', 'age', 'sex',\n",
    "    'household_id', 'district_id',\n",
    "    'economic_status', 'economic_activity_location_id',\n",
    "    # 'school_id_district', \n",
    "    'school_goers',\n",
    "    'manufacturing_workers',\n",
    "    # 'mining_district_id'\n",
    "    # 'household', 'district'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 40 s\n"
     ]
    }
   ],
   "source": [
    "mining_df = expanded_individual_df.copy()\n",
    "\n",
    "p = mining_df['economic_status']\n",
    "\n",
    "mining_df.loc[p == 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p == 'Disabled and not working', 'household_id']\n",
    "mining_df.loc[p != 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p != 'Disabled and not working', 'new_district_id']\n",
    "\n",
    "try:\n",
    "    mining_df.drop('person_id', axis=1, inplace=True)\n",
    "except KeyError:\n",
    "    pass\n",
    "\n",
    "mining_df['person_id'] = mining_df.index\n",
    "mining_df['age'] = mining_df['age'].astype(int)\n",
    "mining_df['economic_status'] = mining_df['economic_status'].str.strip()\n",
    "mining_df.rename(columns={'new_district_id': 'district_id'}, inplace=True)\n",
    "\n",
    "mining_df[relevant_cols].to_pickle(\n",
    "    get_data_dir('preprocessed', 'census', 'zimbabwe_ipums_mining_manufacturing_school_new_dist_100pct.pickle'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>person_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>household_id</th>\n",
       "      <th>district_id</th>\n",
       "      <th>economic_status</th>\n",
       "      <th>economic_activity_location_id</th>\n",
       "      <th>school_goers</th>\n",
       "      <th>manufacturing_workers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>female</td>\n",
       "      <td>h_67786</td>\n",
       "      <td>w_1</td>\n",
       "      <td>Service Workers</td>\n",
       "      <td>w_1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "      <td>male</td>\n",
       "      <td>h_64925</td>\n",
       "      <td>w_1</td>\n",
       "      <td>Service Workers</td>\n",
       "      <td>w_1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>female</td>\n",
       "      <td>h_3648785</td>\n",
       "      <td>w_1</td>\n",
       "      <td>Office workers</td>\n",
       "      <td>w_1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>54</td>\n",
       "      <td>female</td>\n",
       "      <td>h_127965</td>\n",
       "      <td>w_1</td>\n",
       "      <td>Homemakers/Housework</td>\n",
       "      <td>w_1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>65</td>\n",
       "      <td>female</td>\n",
       "      <td>h_103824</td>\n",
       "      <td>w_1</td>\n",
       "      <td>Not working, inactive, not in universe</td>\n",
       "      <td>w_1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   person_id  age     sex household_id district_id  \\\n",
       "0          0   19  female      h_67786     w_1   \n",
       "1          1   49    male      h_64925     w_1   \n",
       "2          2   26  female    h_3648785     w_1   \n",
       "3          3   54  female     h_127965     w_1   \n",
       "4          4   65  female     h_103824     w_1   \n",
       "\n",
       "                          economic_status economic_activity_location_id  \\\n",
       "0                         Service Workers                           w_1   \n",
       "1                         Service Workers                           w_1   \n",
       "2                          Office workers                           w_1   \n",
       "3                    Homemakers/Housework                           w_1   \n",
       "4  Not working, inactive, not in universe                           w_1   \n",
       "\n",
       "   school_goers  manufacturing_workers  \n",
       "0             0                    0.0  \n",
       "1             0                    0.0  \n",
       "2             1                    0.0  \n",
       "3             0                    0.0  \n",
       "4             0                    0.0  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 790 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df[relevant_cols].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Contingent on available data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.1 s\n"
     ]
    }
   ],
   "source": [
    "# mining_df['serial_expanded'] = mining_df['serial'].map(lambda x: f'{x}_01')\n",
    "# mining_df['household_id'] = mining_df['serial_expanded'].map(lambda x: f'h_{x}')\n",
    "mining_df.loc[mining_df['mining_district_id'] != '', 'mining_district_id'] = mining_df.loc[mining_df['mining_district_id'] != '', 'mining_district_id'].map(lambda x: f'mining_{x}')\n",
    "mining_df.loc[mining_df['school_id_district'] != '', 'school_id_district'] = mining_df.loc[mining_df['school_id_district'] != '', 'school_id_district'].map(lambda x: f'school_{x}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serial</th>\n",
       "      <th>persons</th>\n",
       "      <th>district_id</th>\n",
       "      <th>new_district_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>teachers</th>\n",
       "      <th>nfams</th>\n",
       "      <th>isco88a</th>\n",
       "      <th>famunit</th>\n",
       "      <th>...</th>\n",
       "      <th>dhs_ipumsi_zw</th>\n",
       "      <th>province_id_shpfile</th>\n",
       "      <th>province_name_shpfile</th>\n",
       "      <th>district_name_shpfile</th>\n",
       "      <th>serial_expanded</th>\n",
       "      <th>household_id</th>\n",
       "      <th>old_district_id</th>\n",
       "      <th>district_id</th>\n",
       "      <th>economic_activity_location_id</th>\n",
       "      <th>person_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>147171</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>34</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>147171</td>\n",
       "      <td>h_147171</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>21490</td>\n",
       "      <td>2</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>21490</td>\n",
       "      <td>h_21490</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>59596</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>cashiers, tellers and related clerks</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>59596</td>\n",
       "      <td>h_59596</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>148162</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>male</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>miners, shotfirers, stone cutters and carvers</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>148162</td>\n",
       "      <td>h_148162</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>149155</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>female</td>\n",
       "      <td>0</td>\n",
       "      <td>1 family</td>\n",
       "      <td>niu (not in universe)</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>1</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>Bulawayo</td>\n",
       "      <td>149155</td>\n",
       "      <td>h_149155</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   serial  persons district_id  new_district_id  age     sex teachers  \\\n",
       "0  147171        8          21                1   34  female        0   \n",
       "1   21490        2          21                1   36  female        0   \n",
       "2   59596        3          21                1   25  female        0   \n",
       "3  148162        8          21                1   33    male        0   \n",
       "4  149155        8          21                1   18  female        0   \n",
       "\n",
       "      nfams                                        isco88a  famunit  ...  \\\n",
       "0  1 family                          niu (not in universe)        1  ...   \n",
       "1  1 family                          niu (not in universe)        1  ...   \n",
       "2  1 family           cashiers, tellers and related clerks        1  ...   \n",
       "3  1 family  miners, shotfirers, stone cutters and carvers        1  ...   \n",
       "4  1 family                          niu (not in universe)        1  ...   \n",
       "\n",
       "  dhs_ipumsi_zw province_id_shpfile province_name_shpfile  \\\n",
       "0      bulawayo                   1              Bulawayo   \n",
       "1      bulawayo                   1              Bulawayo   \n",
       "2      bulawayo                   1              Bulawayo   \n",
       "3      bulawayo                   1              Bulawayo   \n",
       "4      bulawayo                   1              Bulawayo   \n",
       "\n",
       "  district_name_shpfile serial_expanded household_id old_district_id district_id  \\\n",
       "0              Bulawayo          147171     h_147171        w_21     w_1   \n",
       "1              Bulawayo           21490      h_21490        w_21     w_1   \n",
       "2              Bulawayo           59596      h_59596        w_21     w_1   \n",
       "3              Bulawayo          148162     h_148162        w_21     w_1   \n",
       "4              Bulawayo          149155     h_149155        w_21     w_1   \n",
       "\n",
       "  economic_activity_location_id person_id  \n",
       "0                           w_1         0  \n",
       "1                           w_1         1  \n",
       "2                           w_1         2  \n",
       "3                           w_1         3  \n",
       "4                           w_1         4  \n",
       "\n",
       "[5 rows x 64 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 21.7 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 877 ms\n"
     ]
    }
   ],
   "source": [
    "p = mining_df['economic_status']\n",
    "\n",
    "mining_df.loc[p == 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p == 'Disabled and not working', 'household_id']\n",
    "mining_df.loc[p != 'Disabled and not working', 'economic_activity_location_id'] = mining_df.loc[p != 'Disabled and not working', 'new_district_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.21 s\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    mining_df.drop('person_id', axis=1, inplace=True)\n",
    "except KeyError:\n",
    "    pass\n",
    "\n",
    "mining_df['person_id'] = mining_df.index\n",
    "mining_df['age'] = mining_df['age'].astype(int)\n",
    "mining_df['economic_status'] = mining_df['economic_status'].str.strip()\n",
    "mining_df.rename(columns={'new_district_id': 'district_id'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 870 µs\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 932 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df[relevant_cols][\n",
    "    mining_df.serial_expanded.str.endswith('_01')\n",
    "].to_pickle('../../data/zimbabwe_ipums_mining_manufacturing_school_new_dist_5pct.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 1.73 s\n"
     ]
    }
   ],
   "source": [
    "mining_df[relevant_cols][\n",
    "    mining_df.serial_expanded.str.endswith('_01') | mining_df.serial_expanded.str.endswith('_02')\n",
    "].to_pickle('../../data/zimbabwe_ipums_mining_manufacturing_school_new_dist_10pct.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1309376, 69)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.32 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mining_df = pd.read_pickle('../../data/zimbabwe_ipums_mining_manufacturing_school_new_dist_10pct.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1              1\n",
       "2              2\n",
       "3              3\n",
       "10            10\n",
       "20            20\n",
       "           ...  \n",
       "654638    654638\n",
       "654646    654646\n",
       "654647    654647\n",
       "654656    654656\n",
       "654682    654682\n",
       "Name: person_id, Length: 126990, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mining_df[mining_df['school_id_district'] != '']['person_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Expand to 10%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serial</th>\n",
       "      <th>persons</th>\n",
       "      <th>hhwt</th>\n",
       "      <th>subsamp</th>\n",
       "      <th>strata</th>\n",
       "      <th>gq</th>\n",
       "      <th>urban</th>\n",
       "      <th>geolev1</th>\n",
       "      <th>Province</th>\n",
       "      <th>Province_str</th>\n",
       "      <th>...</th>\n",
       "      <th>manufacturing_workers</th>\n",
       "      <th>mining_workers</th>\n",
       "      <th>mining_districts</th>\n",
       "      <th>mining_district_id</th>\n",
       "      <th>serial_expanded</th>\n",
       "      <th>household_id</th>\n",
       "      <th>old_district_id</th>\n",
       "      <th>district_id</th>\n",
       "      <th>economic_activity_location_id</th>\n",
       "      <th>person_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4908000</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>10th 1% subsample</td>\n",
       "      <td>12552</td>\n",
       "      <td>households</td>\n",
       "      <td>urban</td>\n",
       "      <td>716000</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>4908000_01</td>\n",
       "      <td>h_4908000_01</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>944000</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>44th 1% subsample</td>\n",
       "      <td>893</td>\n",
       "      <td>households</td>\n",
       "      <td>urban</td>\n",
       "      <td>716000</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>944000_01</td>\n",
       "      <td>h_944000_01</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5669000</td>\n",
       "      <td>4</td>\n",
       "      <td>20</td>\n",
       "      <td>9th 1% subsample</td>\n",
       "      <td>6838</td>\n",
       "      <td>households</td>\n",
       "      <td>urban</td>\n",
       "      <td>716000</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>5669000_01</td>\n",
       "      <td>h_5669000_01</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3231000</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>37th 1% subsample</td>\n",
       "      <td>9404</td>\n",
       "      <td>households</td>\n",
       "      <td>urban</td>\n",
       "      <td>716000</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>3231000_01</td>\n",
       "      <td>h_3231000_01</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1544000</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>44th 1% subsample</td>\n",
       "      <td>3738</td>\n",
       "      <td>households</td>\n",
       "      <td>urban</td>\n",
       "      <td>716000</td>\n",
       "      <td>0</td>\n",
       "      <td>bulawayo</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>1544000_01</td>\n",
       "      <td>h_1544000_01</td>\n",
       "      <td>w_21</td>\n",
       "      <td>w_1</td>\n",
       "      <td>w_1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654683</th>\n",
       "      <td>107627000</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>4th 1% subsample</td>\n",
       "      <td>1741</td>\n",
       "      <td>households</td>\n",
       "      <td>rural</td>\n",
       "      <td>716007</td>\n",
       "      <td>7</td>\n",
       "      <td>midlands</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>107627000_01</td>\n",
       "      <td>h_107627000_01</td>\n",
       "      <td>w_708</td>\n",
       "      <td>w_60</td>\n",
       "      <td>w_60</td>\n",
       "      <td>654683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654684</th>\n",
       "      <td>112072000</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>62nd 1% subsample</td>\n",
       "      <td>2077</td>\n",
       "      <td>households</td>\n",
       "      <td>urban</td>\n",
       "      <td>716007</td>\n",
       "      <td>7</td>\n",
       "      <td>midlands</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>112072000_01</td>\n",
       "      <td>h_112072000_01</td>\n",
       "      <td>w_724</td>\n",
       "      <td>w_60</td>\n",
       "      <td>w_60</td>\n",
       "      <td>654684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654685</th>\n",
       "      <td>107362000</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>5th 1% subsample</td>\n",
       "      <td>13286</td>\n",
       "      <td>households</td>\n",
       "      <td>rural</td>\n",
       "      <td>716007</td>\n",
       "      <td>7</td>\n",
       "      <td>midlands</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>107362000_01</td>\n",
       "      <td>h_107362000_01</td>\n",
       "      <td>w_708</td>\n",
       "      <td>w_60</td>\n",
       "      <td>w_60</td>\n",
       "      <td>654685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654686</th>\n",
       "      <td>107396000</td>\n",
       "      <td>8</td>\n",
       "      <td>20</td>\n",
       "      <td>79th 1% subsample</td>\n",
       "      <td>4246</td>\n",
       "      <td>households</td>\n",
       "      <td>rural</td>\n",
       "      <td>716007</td>\n",
       "      <td>7</td>\n",
       "      <td>midlands</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>107396000_01</td>\n",
       "      <td>h_107396000_01</td>\n",
       "      <td>w_708</td>\n",
       "      <td>w_60</td>\n",
       "      <td>w_60</td>\n",
       "      <td>654686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654687</th>\n",
       "      <td>107794000</td>\n",
       "      <td>7</td>\n",
       "      <td>20</td>\n",
       "      <td>28th 1% subsample</td>\n",
       "      <td>2173</td>\n",
       "      <td>households</td>\n",
       "      <td>rural</td>\n",
       "      <td>716007</td>\n",
       "      <td>7</td>\n",
       "      <td>midlands</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>107794000_01</td>\n",
       "      <td>h_107794000_01</td>\n",
       "      <td>w_708</td>\n",
       "      <td>w_60</td>\n",
       "      <td>w_60</td>\n",
       "      <td>654687</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>654688 rows × 69 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           serial  persons  hhwt            subsamp  strata          gq  \\\n",
       "0         4908000        4    20  10th 1% subsample   12552  households   \n",
       "1          944000        5    20  44th 1% subsample     893  households   \n",
       "2         5669000        4    20   9th 1% subsample    6838  households   \n",
       "3         3231000        8    20  37th 1% subsample    9404  households   \n",
       "4         1544000        6    20  44th 1% subsample    3738  households   \n",
       "...           ...      ...   ...                ...     ...         ...   \n",
       "654683  107627000        8    20   4th 1% subsample    1741  households   \n",
       "654684  112072000        6    20  62nd 1% subsample    2077  households   \n",
       "654685  107362000        8    20   5th 1% subsample   13286  households   \n",
       "654686  107396000        8    20  79th 1% subsample    4246  households   \n",
       "654687  107794000        7    20  28th 1% subsample    2173  households   \n",
       "\n",
       "        urban  geolev1  Province Province_str  ...  manufacturing_workers  \\\n",
       "0       urban   716000         0     bulawayo  ...                    NaN   \n",
       "1       urban   716000         0     bulawayo  ...                    NaN   \n",
       "2       urban   716000         0     bulawayo  ...                    NaN   \n",
       "3       urban   716000         0     bulawayo  ...                    NaN   \n",
       "4       urban   716000         0     bulawayo  ...                    NaN   \n",
       "...       ...      ...       ...          ...  ...                    ...   \n",
       "654683  rural   716007         7     midlands  ...                    NaN   \n",
       "654684  urban   716007         7     midlands  ...                    NaN   \n",
       "654685  rural   716007         7     midlands  ...                    NaN   \n",
       "654686  rural   716007         7     midlands  ...                    NaN   \n",
       "654687  rural   716007         7     midlands  ...                    NaN   \n",
       "\n",
       "        mining_workers  mining_districts  mining_district_id  serial_expanded  \\\n",
       "0                  NaN               NaN                           4908000_01   \n",
       "1                  NaN               NaN                            944000_01   \n",
       "2                  NaN               NaN                           5669000_01   \n",
       "3                  NaN               NaN                           3231000_01   \n",
       "4                  NaN               NaN                           1544000_01   \n",
       "...                ...               ...                 ...              ...   \n",
       "654683             NaN               NaN                         107627000_01   \n",
       "654684             NaN               NaN                         112072000_01   \n",
       "654685             NaN               NaN                         107362000_01   \n",
       "654686             NaN               NaN                         107396000_01   \n",
       "654687             NaN               NaN                         107794000_01   \n",
       "\n",
       "          household_id  old_district_id  district_id  economic_activity_location_id  \\\n",
       "0         h_4908000_01         w_21      w_1                            w_1   \n",
       "1          h_944000_01         w_21      w_1                            w_1   \n",
       "2         h_5669000_01         w_21      w_1                            w_1   \n",
       "3         h_3231000_01         w_21      w_1                            w_1   \n",
       "4         h_1544000_01         w_21      w_1                            w_1   \n",
       "...                ...          ...      ...                            ...   \n",
       "654683  h_107627000_01        w_708     w_60                           w_60   \n",
       "654684  h_112072000_01        w_724     w_60                           w_60   \n",
       "654685  h_107362000_01        w_708     w_60                           w_60   \n",
       "654686  h_107396000_01        w_708     w_60                           w_60   \n",
       "654687  h_107794000_01        w_708     w_60                           w_60   \n",
       "\n",
       "       person_id  \n",
       "0              0  \n",
       "1              1  \n",
       "2              2  \n",
       "3              3  \n",
       "4              4  \n",
       "...          ...  \n",
       "654683    654683  \n",
       "654684    654684  \n",
       "654685    654685  \n",
       "654686    654686  \n",
       "654687    654687  \n",
       "\n",
       "[654688 rows x 69 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 261 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   298,    315,    325, ..., 654637, 654654, 654659])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 143 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df[~mining_df['household_id'].str.startswith('h_')]['person_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5              5\n",
       "95            95\n",
       "251          251\n",
       "274          274\n",
       "302          302\n",
       "           ...  \n",
       "654084    654084\n",
       "654108    654108\n",
       "654170    654170\n",
       "654335    654335\n",
       "654571    654571\n",
       "Name: person_id, Length: 10471, dtype: int64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 16.8 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df[mining_df.manufacturing_workers.notnull()].person_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([     5,     95,    251, ..., 654170, 654335, 654571])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 16.3 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df[mining_df['manufacturing_workers'].notnull()]['person_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(63495,)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 46.2 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df[mining_df['school_id_district'] != '']['person_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(654688,)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 2.91 ms\n"
     ]
    }
   ],
   "source": [
    "mining_df['school_id_district'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "wb-nlp",
   "language": "python",
   "name": "wb-nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}