update model card example to create better pre-processing function

sassoftware · Dec 23, 2024 · 5306aee · 5306aee
1 parent e2bd4e5
commit 5306aee
Showing 1 changed file with 16 additions and 2 deletions.
diff --git a/examples/pzmm_generate_complete_model_card.ipynb b/examples/pzmm_generate_complete_model_card.ipynb
@@ -568,7 +568,8 @@
     "    df.columns = df.columns.str.replace(' ', '')\n",
     "    df.columns = df.columns.str.replace('-', '_')\n",
     "    df = df.drop(['Sex_Male'], axis=1)\n",
-    "    df = pd.concat([df, cat_vals], axis=1).drop('index', axis=1)\n",
+    "    if 'index' in df.columns or 'index' in cat_vals.columns:\n",
+    "        df = pd.concat([df, cat_vals], axis=1).drop('index', axis=1)\n",
     "    # For the model to score correctly, all OHE columns must exist\n",
     "    input_cols = [\n",
     "        \"Education_9th\", \"Education_10th\", \"Education_11th\", \"Education_12th\", \"Education_Assoc_voc\", \"Education_Assoc_acdm\", \"Education_Masters\", \"Education_Prof_school\",\n",
@@ -579,9 +580,20 @@
     "        'Relationship_Not_in_family', 'Relationship_Own_child', 'Relationship_Unmarried', 'Relationship_Wife', 'Relationship_Other_relative', 'WorkClass_Private',\n",
     "        'Education_Bachelors'\n",
     "    ]\n",
+    "    # OHE columns must be removed after data combination\n",
+    "    predictor_columns = ['Age', 'HoursPerWeek', 'WorkClass_Private', 'WorkClass_Self', 'WorkClass_Gov', \n",
+    "       'WorkClass_Other', 'Education_HS_grad', 'Education_Some_HS', 'Education_Assoc', 'Education_Some_college',\n",
+    "       'Education_Bachelors', 'Education_Adv_Degree', 'Education_No_HS', 'MartialStatus_Married_civ_spouse',\n",
+    "       'MartialStatus_Never_married', 'MartialStatus_Divorced', 'MartialStatus_Separated', 'MartialStatus_Widowed',\n",
+    "       'MartialStatus_Other', 'Relationship_Husband', 'Relationship_Not_in_family', 'Relationship_Own_child', 'Relationship_Unmarried',\n",
+    "       'Relationship_Wife', 'Relationship_Other_relative', 'Race_White', 'Race_Black', 'Race_Asian_Pac_Islander',\n",
+    "       'Race_Amer_Indian_Eskimo', 'Race_Other', 'Sex_Female']\n",
+    "\n",
     "    for col in input_cols:\n",
     "        if col not in df.columns:\n",
     "            df[col] = 0\n",
+    "    \n",
+    "\n",
     "    df[\"Education_Some_HS\"] = df[\"Education_9th\"] | df[\"Education_10th\"] | df[\"Education_11th\"] | df[\"Education_12th\"]\n",
     "    df[\"Education_Assoc\"] = df[\"Education_Assoc_voc\"] | df[\"Education_Assoc_acdm\"]\n",
     "    df[\"Education_Adv_Degree\"] = df[\"Education_Masters\"] | df[\"Education_Prof_school\"] | df[\"Education_Doctorate\"]\n",
@@ -593,6 +605,8 @@
     "\n",
     "    df[\"MartialStatus_Other\"] = df[\"MartialStatus_Married_spouse_absent\"] | df[\"MartialStatus_Married_AF_spouse\"]\n",
     "\n",
+    "    df = df[predictor_columns]\n",
+    "\n",
     "    return df"
    ]
   },
@@ -1772,7 +1786,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "pandatwo",
    "language": "python",
    "name": "python3"
   },