2-training-inference/2.3-automl-training/local-compute/binayclassification-employee-attrition-autoaml-local-compute.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AutoML Classification experiment using Local Compute and Pandas DataFrames\n",
    "\n",
    "- Save trained model as Scikit-Learn model (.pkl) and as ONNX model (.onnx file)\n",
    "- Data: IBM Employee Attrition dataset loaded from Azure ML Dataset\n",
    "\n",
    "**Note:** Install `onnxruntime` via pip and then restart the kernel of this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install onnxruntime\n",
    "# Restart Kernel afterwards!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Get Azure ML Workspace to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check core SDK version number\n",
    "import azureml.core\n",
    "\n",
    "print(\"SDK version:\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Workspace, Dataset\n",
    "\n",
    "# Get Workspace defined in by default config.json file\n",
    "ws = Workspace.from_config()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data from Azure ML Datasets into Pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Data\n",
    "aml_dataset = ws.datasets['IBM-Employee-Attrition']\n",
    "\n",
    "# Use Pandas DataFrame just to sneak peak some data and schema\n",
    "full_df = aml_dataset.to_pandas_dataframe()\n",
    "# .to_pandas_dataframe().dropna()\n",
    "full_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use Pandas DataFrame just to investigate the dataset's schema and info\n",
    "full_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean up the initial dataset (Using related Pandas DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping Employee count as all values are 1 and hence attrition is independent of this feature\n",
    "full_df = full_df.drop(['EmployeeCount'], axis=1)\n",
    "\n",
    "# Dropping Employee Number since it is merely an identifier\n",
    "full_df = full_df.drop(['EmployeeNumber'], axis=1)\n",
    "\n",
    "full_df = full_df.drop(['Over18'], axis=1)\n",
    "\n",
    "# Since all values are 80\n",
    "full_df = full_df.drop(['StandardHours'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split original dataset in test/train sets using Scikit-Learn train_test_split function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Split using ScikitLearn train_test_split function using Dataframes\n",
    "# Will use test dataframe at the end, without AutoML, just for testing predictions with the model\n",
    "\n",
    "# Only split in test/train\n",
    "train_df, test_df = train_test_split(full_df, test_size=0.2, random_state=1)\n",
    "train_df.describe()\n",
    "\n",
    "# Split in x/y and test/train\n",
    "# y_df = full_df.pop(\"Attrition\")\n",
    "# x_df = full_df\n",
    "# x_train, x_test, y_train, y_test = train_test_split(x_df, y_df, test_size=0.1, random_state=1)\n",
    "\n",
    "#Another possibility would be to split using the Azure ML Datasets (Better for Remote Compute): \n",
    "# https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.data.tabulardataset?view=azure-ml-py#random-split-percentage--seed-none-"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## List and select primary metric to drive the AutoML classification problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.train import automl\n",
    "\n",
    "# List of possible primary metrics is here:\n",
    "# https://docs.microsoft.com/en-us/azure/machine-learning/how-to-configure-auto-train#primary-metric\n",
    "    \n",
    "# Get a list of valid metrics for your given task\n",
    "automl.utilities.get_primary_metrics('classification')\n",
    "\n",
    "# We'll use 'accuracy' as primary metric (Closer to 1.00 is better)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define AutoML Experiment settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "# Explanation of Settings: https://docs.microsoft.com/en-us/azure/machine-learning/how-to-configure-auto-train#configure-your-experiment-settings\n",
    "\n",
    "# AutoMLConfig info on: \n",
    "# https://docs.microsoft.com/en-us/python/api/azureml-train-automl-client/azureml.train.automl.automlconfig.automlconfig\n",
    "\n",
    "# You can provide additional settings as a **kwargs parameter for the AutoMLConfig object\n",
    "# automl_settings = {\n",
    "#     \"whitelist_models\": 'XGBoostClassifier'\n",
    "# }\n",
    "\n",
    "from azureml.train.automl import AutoMLConfig\n",
    "\n",
    "project_folder = './automl'\n",
    "os.makedirs(project_folder, exist_ok=True)\n",
    "\n",
    "automl_config = AutoMLConfig(task='classification',\n",
    "                             primary_metric='accuracy',\n",
    "                             experiment_timeout_minutes=5,\n",
    "                             training_data=train_df,\n",
    "                             label_column_name=\"Attrition\",\n",
    "                             n_cross_validations=5,\n",
    "                             # blacklist_models='XGBoostClassifier', \n",
    "                             # iteration_timeout_minutes=5,                                                    \n",
    "                             enable_early_stopping= True,\n",
    "                             featurization='auto',\n",
    "                             debug_log='automated_ml_errors.log',\n",
    "                             verbosity=logging.INFO,\n",
    "                             enable_onnx_compatible_models=True,\n",
    "                             path=project_folder\n",
    "                             # **automl_settings\n",
    "                             )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Experiment with multiple child runs under the covers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Experiment\n",
    "from datetime import datetime\n",
    "\n",
    "now = datetime.now()\n",
    "time_string = now.strftime(\"%m-%d-%Y-%H\")\n",
    "print(time_string)\n",
    "experiment_name = \"classif-automl-local-{0}\".format(time_string)\n",
    "print(experiment_name)\n",
    "\n",
    "experiment = Experiment(workspace=ws, name=experiment_name)\n",
    "\n",
    "run = experiment.submit(automl_config, show_output=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore results with Widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Explore the results of automatic training with a Jupyter widget: https://docs.microsoft.com/en-us/python/api/azureml-widgets/azureml.widgets?view=azure-ml-py\n",
    "from azureml.widgets import RunDetails\n",
    "RunDetails(run).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve the 'Best' Scikit-Learn Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, fitted_model = run.get_output()\n",
    "print(best_run)\n",
    "print('--------')\n",
    "print(fitted_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve the 'Best' ONNX Model\n",
    "Below we select the best pipeline from our iterations. The get_output method returns the best run and the fitted model. The Model includes the pipeline and any pre-processing. Overloads on get_output allow you to retrieve the best run and fitted model for any logged metric or for a particular iteration.\n",
    "Set the parameter return_onnx_model=True to retrieve the best ONNX model, instead of the Python model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, onnx_mdl = run.get_output(return_onnx_model=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicetely Save the best ONNX model on local drive path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.automl.runtime.onnx_convert import OnnxConverter\n",
    "onnx_fl_path = \"./best_model.onnx\"\n",
    "OnnxConverter.save_onnx_model(onnx_mdl, onnx_fl_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## See files associated with the 'Best run'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(best_run.get_file_names())\n",
    "\n",
    "# best_run.download_file('azureml-logs/70_driver_log.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download experiment run related files \n",
    "Model files (.pkl and .onnx), Environment files to see Conda and Environment dependencies used by AutoML, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the run's files\n",
    "best_run.download_file('outputs/model.pkl')\n",
    "best_run.download_file('outputs/model.onnx')\n",
    "best_run.download_file('outputs/model_onnx.json')\n",
    "best_run.download_file('outputs/conda_env_v_1_0_0.yml')\n",
    "best_run.download_file('outputs/env_dependencies.json')\n",
    "best_run.download_file('outputs/scoring_file_v_1_0_0.py')\n",
    "best_run.download_file('pipeline_graph.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Register the Scikit-Learn model (.pkl file)\n",
    "Once you've trained the model, you can save and register it to your workspace. Model registration lets you store and version your models in your workspace to simplify model management and deployment.\n",
    "\n",
    "Running the following code will register the model to your workspace, and will make it available to reference by name in remote compute contexts or deployment scripts. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core.model import Model\n",
    "\n",
    "model_reg = best_run.register_model(model_name='classif-empl-automl-local',      # Name of the registered model in your workspace.\n",
    "                                    description='Binary classification model for employees attrition. From AutoML local training',\n",
    "                                    model_path='outputs/model.pkl',              # Local file to upload and register as a model.\n",
    "                                    model_framework=Model.Framework.SCIKITLEARN, # Framework used to create the model.\n",
    "                                    model_framework_version='0.20.3',            # Version of scikit-learn used to create the model.\n",
    "                                    tags={'ml-task': \"binary-classification\", 'business-area': \"HR\"},\n",
    "                                    properties={'pandas-version': \"0.23.4\"},\n",
    "                                    sample_input_dataset=aml_dataset\n",
    "                                  )\n",
    "\n",
    "print(model_reg)\n",
    "\n",
    "# (Q1:) How can we know what frameworks/libraries versions were used by AutoML when training the model?\n",
    "#         - If investigating the outputs/conda_env_v_1_0_0.yml file, some libraries like Pandas and Scikit-Learn are not showing any version...\n",
    "\n",
    "# (Q2:) Why best_run.get_environment() fails?\n",
    "# best_run_environment = best_run.get_environment() \n",
    "# print(best_run_environment)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prep Test Data: Extract X values (feature columns) from test dataset and convert to NumPi array for predicting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#Remove Label/y column\n",
    "if 'Attrition' in test_df.columns:\n",
    "    y_test_df = test_df.pop('Attrition')\n",
    "\n",
    "x_test_df = test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Predictions with Scikit-Learn Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### (Optional) Download Model from Registry and load in-memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(Model.get_model_path('classif-empl-automl-local', _workspace=ws))\n",
    "\n",
    "model_definition_from_registry = Model(ws,'classif-empl-automl-local')\n",
    "model_definition_from_registry.download(target_dir='.', exist_ok=True)\n",
    "print(model_definition_from_registry)\n",
    "print('-------')\n",
    "\n",
    "# Load the model into memory\n",
    "import joblib\n",
    "fitted_model = joblib.load('model.pkl')\n",
    "print(fitted_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try the best model making predictions with the test dataset\n",
    "y_predictions = fitted_model.predict(x_test_df)\n",
    "\n",
    "print('10 predictions: ')\n",
    "print(y_predictions[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predictions.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Predictions with the ONNX model, using onnxruntime package\n",
    "Needs pip install onnxruntime==1.0.0' in environment (Also try with 1.1.0 version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import json\n",
    "from azureml.automl.core.onnx_convert import OnnxConvertConstants\n",
    "from azureml.train.automl import constants\n",
    "\n",
    "if sys.version_info < OnnxConvertConstants.OnnxIncompatiblePythonVersion:\n",
    "    python_version_compatible = True\n",
    "else:\n",
    "    python_version_compatible = False\n",
    "    \n",
    "import onnxruntime\n",
    "from azureml.automl.runtime.onnx_convert import OnnxInferenceHelper\n",
    "\n",
    "def get_onnx_res(run):\n",
    "    res_path = 'onnx_resource.json'\n",
    "    run.download_file(name=constants.MODEL_RESOURCE_PATH_ONNX, output_file_path=res_path)\n",
    "    with open(res_path) as f:\n",
    "        onnx_res = json.load(f)\n",
    "    return onnx_res\n",
    "\n",
    "if python_version_compatible:\n",
    "    # test_df = test_dataset.to_pandas_dataframe()\n",
    "    mdl_bytes = onnx_mdl.SerializeToString()\n",
    "    onnx_res = get_onnx_res(best_run)\n",
    "\n",
    "    onnxrt_helper = OnnxInferenceHelper(mdl_bytes, onnx_res)\n",
    "    pred_onnx, pred_prob_onnx = onnxrt_helper.predict(x_test_df)\n",
    "\n",
    "    print('Predicting with ONNX model...')\n",
    "    print(pred_onnx)\n",
    "    print(pred_prob_onnx)\n",
    "else:\n",
    "    print('Please use Python version 3.6 or 3.7 to run the inference helper.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate the Accuracy with Test Dataset (Data not used for training)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print('Accuracy with Scikit-Learn model:')\n",
    "print(accuracy_score(y_test_df, y_predictions))\n",
    "\n",
    "print('Accuracy with ONNX model:')\n",
    "print(accuracy_score(y_test_df, pred_onnx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}