From 5306aee8105ca8ba187b20ab3cecea9b10fa763a Mon Sep 17 00:00:00 2001 From: djm21 Date: Mon, 23 Dec 2024 11:47:42 -0500 Subject: [PATCH] update model card example to create better pre-processing function --- .../pzmm_generate_complete_model_card.ipynb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/examples/pzmm_generate_complete_model_card.ipynb b/examples/pzmm_generate_complete_model_card.ipynb index 7ebf0858..87fb8f63 100644 --- a/examples/pzmm_generate_complete_model_card.ipynb +++ b/examples/pzmm_generate_complete_model_card.ipynb @@ -568,7 +568,8 @@ " df.columns = df.columns.str.replace(' ', '')\n", " df.columns = df.columns.str.replace('-', '_')\n", " df = df.drop(['Sex_Male'], axis=1)\n", - " df = pd.concat([df, cat_vals], axis=1).drop('index', axis=1)\n", + " if 'index' in df.columns or 'index' in cat_vals.columns:\n", + " df = pd.concat([df, cat_vals], axis=1).drop('index', axis=1)\n", " # For the model to score correctly, all OHE columns must exist\n", " input_cols = [\n", " \"Education_9th\", \"Education_10th\", \"Education_11th\", \"Education_12th\", \"Education_Assoc_voc\", \"Education_Assoc_acdm\", \"Education_Masters\", \"Education_Prof_school\",\n", @@ -579,9 +580,20 @@ " 'Relationship_Not_in_family', 'Relationship_Own_child', 'Relationship_Unmarried', 'Relationship_Wife', 'Relationship_Other_relative', 'WorkClass_Private',\n", " 'Education_Bachelors'\n", " ]\n", + " # OHE columns must be removed after data combination\n", + " predictor_columns = ['Age', 'HoursPerWeek', 'WorkClass_Private', 'WorkClass_Self', 'WorkClass_Gov', \n", + " 'WorkClass_Other', 'Education_HS_grad', 'Education_Some_HS', 'Education_Assoc', 'Education_Some_college',\n", + " 'Education_Bachelors', 'Education_Adv_Degree', 'Education_No_HS', 'MartialStatus_Married_civ_spouse',\n", + " 'MartialStatus_Never_married', 'MartialStatus_Divorced', 'MartialStatus_Separated', 'MartialStatus_Widowed',\n", + " 'MartialStatus_Other', 'Relationship_Husband', 'Relationship_Not_in_family', 'Relationship_Own_child', 'Relationship_Unmarried',\n", + " 'Relationship_Wife', 'Relationship_Other_relative', 'Race_White', 'Race_Black', 'Race_Asian_Pac_Islander',\n", + " 'Race_Amer_Indian_Eskimo', 'Race_Other', 'Sex_Female']\n", + "\n", " for col in input_cols:\n", " if col not in df.columns:\n", " df[col] = 0\n", + " \n", + "\n", " df[\"Education_Some_HS\"] = df[\"Education_9th\"] | df[\"Education_10th\"] | df[\"Education_11th\"] | df[\"Education_12th\"]\n", " df[\"Education_Assoc\"] = df[\"Education_Assoc_voc\"] | df[\"Education_Assoc_acdm\"]\n", " df[\"Education_Adv_Degree\"] = df[\"Education_Masters\"] | df[\"Education_Prof_school\"] | df[\"Education_Doctorate\"]\n", @@ -593,6 +605,8 @@ "\n", " df[\"MartialStatus_Other\"] = df[\"MartialStatus_Married_spouse_absent\"] | df[\"MartialStatus_Married_AF_spouse\"]\n", "\n", + " df = df[predictor_columns]\n", + "\n", " return df" ] }, @@ -1772,7 +1786,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "pandatwo", "language": "python", "name": "python3" },