examples/SKLEARN-TO-JSON.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://github.com/FireFlyTy/datrics-model-json\n",
      "  Cloning https://github.com/FireFlyTy/datrics-model-json to /private/var/folders/vd/c6p08wcd38l_mvcjcy4g9h200000gq/T/pip-req-build-1078ux78\n",
      "  Running command git clone -q https://github.com/FireFlyTy/datrics-model-json /private/var/folders/vd/c6p08wcd38l_mvcjcy4g9h200000gq/T/pip-req-build-1078ux78\n",
      "Requirement already satisfied: scikit-learn>=0.21.3 in ./git/pipeline-2/venv/lib/python3.8/site-packages (from datrics-json==0.1.0) (0.23.2)\n",
      "Requirement already satisfied: scipy>=0.19.1 in ./git/pipeline-2/venv/lib/python3.8/site-packages (from scikit-learn>=0.21.3->datrics-json==0.1.0) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in ./git/pipeline-2/venv/lib/python3.8/site-packages (from scikit-learn>=0.21.3->datrics-json==0.1.0) (1.18.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in ./git/pipeline-2/venv/lib/python3.8/site-packages (from scikit-learn>=0.21.3->datrics-json==0.1.0) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in ./git/pipeline-2/venv/lib/python3.8/site-packages (from scikit-learn>=0.21.3->datrics-json==0.1.0) (0.14.1)\n",
      "Building wheels for collected packages: datrics-json\n",
      "  Building wheel for datrics-json (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for datrics-json: filename=datrics_json-0.1.0-py3-none-any.whl size=15888 sha256=cc2bd389faa0942ff943ebea55aaa310256442236e73457e4926d0a19fb6a846\n",
      "  Stored in directory: /private/var/folders/vd/c6p08wcd38l_mvcjcy4g9h200000gq/T/pip-ephem-wheel-cache-gl6wt6eg/wheels/c0/fb/8f/40343db0c6c0743e11aebc583059741b45c1de051eca324e76\n",
      "Successfully built datrics-json\n",
      "Installing collected packages: datrics-json\n",
      "Successfully installed datrics-json-0.1.0\n"
     ]
    }
   ],
   "source": [
    "! pip install git+https://github.com/FireFlyTy/datrics-model-json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import DBSCAN\n",
    "from sklearn.ensemble import IsolationForest, ExtraTreesRegressor, RandomForestRegressor\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.tree import ExtraTreeRegressor, BaseDecisionTree\n",
    "from sklearn.linear_model import LogisticRegression, LinearRegression, ElasticNet, Ridge, Lasso\n",
    "\n",
    "from dask_ml.preprocessing import OneHotEncoder, Categorizer, LabelEncoder, MinMaxScaler\n",
    "import dask.dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgbm\n",
    "import json\n",
    "import numpy as np\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datrics_json as datjson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_model(model, deserealized_model, X):\n",
    "    if isinstance(model, DBSCAN):\n",
    "        return(model.fit_predict(X).tolist() == deserealized_model.fit_predict(X).tolist())\n",
    "    elif isinstance(model, LabelEncoder):\n",
    "        return(model.transform(X).flatten().tolist() == \n",
    "               deserealized_model.transform(X).flatten().tolist())\n",
    "    elif isinstance(model, OneHotEncoder):\n",
    "        return(model.transform(X).compute().values.tolist() == \n",
    "               deserealized_model.transform(X).compute().values.tolist())\n",
    "    elif isinstance(model, MinMaxScaler):\n",
    "        return(model.transform(X).compute().values.tolist() == \n",
    "               deserealized_model.transform(X).compute().values.tolist())\n",
    "    else:\n",
    "        return(model.predict(X).tolist() == deserealized_model.predict(X).tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data_classification = pd.read_csv('iris.csv')\n",
    "data_regression = pd.read_csv('Downloads/online_retail_II.csv')\n",
    "\n",
    "X_class = data_classification[['sepal length', 'sepal width', 'petal length', 'petal width']]\n",
    "y_class = data_classification.target\n",
    "y_class_1 = data_classification.target == 'Iris-setosa'\n",
    "\n",
    "\n",
    "\n",
    "a = data_regression.StockCode.value_counts()>1000\n",
    "data_regression = data_regression[data_regression.StockCode.isin(a.index.values[a])]\n",
    "data_regression.InvoiceDate = data_regression.InvoiceDate.astype('datetime64[ns]')\n",
    "data_regression['date_month'] = data_regression.InvoiceDate.dt.month\n",
    "data_regression['week_day'] = data_regression.InvoiceDate.dt.day_of_week\n",
    "data_regression = data_regression[['StockCode', 'Quantity', 'Price', 'date_month', 'week_day']]\n",
    "data_regression = pd.get_dummies(data_regression, columns = ['StockCode'])\n",
    "X_regr = data_regression.drop('Quantity', axis = 1)\n",
    "y_regr = data_regression['Quantity']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modelling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SKLearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_logreg = LogisticRegression().fit(X_class, y_class)\n",
    "model_linear = LinearRegression().fit(X_regr, y_regr)\n",
    "model_elastic = ElasticNet().fit(X_regr, y_regr)\n",
    "model_ridge = Ridge().fit(X_regr, y_regr)\n",
    "model_lasso = Lasso().fit(X_regr, y_regr)\n",
    "\n",
    "model_kmeans = KMeans(n_clusters=5).fit(X_class)\n",
    "model_dbscan = DBSCAN().fit(X_class)\n",
    "model_iforest = IsolationForest().fit(X_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dask-ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MinMaxScaler()"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yy = dd.from_pandas(pd.DataFrame(y_class), npartitions=2)\n",
    "xx = dd.from_pandas(pd.DataFrame(X_class.iloc[:,0:1]), npartitions=2)\n",
    "\n",
    "encoder_label = LabelEncoder().fit(yy[['target']].categorize())\n",
    "encoder_onehot = OneHotEncoder().fit(yy[['target']].categorize())\n",
    "\n",
    "scaler_min_max = MinMaxScaler()\n",
    "scaler_min_max.fit(xx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGBM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiclass Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init_parameters={}\n",
    "model_init_parameters['boosting'] = 'gbdt' \n",
    "model_init_parameters['objective'] = 'multiclass'  # hardcode for binary classification\n",
    "model_init_parameters['importance_type'] = 'gain'  # hardcode for gain\n",
    "model_init_parameters['learning_rate'] = 0.1\n",
    "model_init_parameters['num_iterations'] = 100\n",
    "model_init_parameters['num_leaves'] = 31\n",
    "\n",
    "model_lgbm_m_classifier = lgbm.LGBMClassifier(**model_init_parameters).fit(X_class, y_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Binary Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init_parameters={}\n",
    "model_init_parameters['boosting'] = 'gbdt' \n",
    "model_init_parameters['objective'] = 'binary'  # hardcode for binary classification\n",
    "model_init_parameters['importance_type'] = 'gain'  # hardcode for gain\n",
    "model_init_parameters['learning_rate'] = 0.1\n",
    "model_init_parameters['num_iterations'] = 100\n",
    "model_init_parameters['num_leaves'] = 31\n",
    "\n",
    "model_lgbm_b_classifier = lgbm.LGBMClassifier(**model_init_parameters).fit(X_class, y_class_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init_parameters={}\n",
    "model_init_parameters['boosting'] = 'gbdt' \n",
    "model_init_parameters['objective'] = 'regression'  # hardcode for binary classification\n",
    "model_init_parameters['importance_type'] = 'gain'  # hardcode for gain\n",
    "model_init_parameters['learning_rate'] = 0.1\n",
    "model_init_parameters['num_iterations'] = 100\n",
    "model_init_parameters['num_leaves'] = 31\n",
    "\n",
    "model_init_parameters['reg_alpha'] = 0.5\n",
    "model_init_parameters['reg_lambda'] = 0.5\n",
    "\n",
    "\n",
    "model_lgbm_regression = lgbm.LGBMRegressor(**model_init_parameters).fit(X_regr, y_regr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiclass Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init_parameters={}\n",
    "model_init_parameters['boosting'] = 'rf' \n",
    "model_init_parameters['objective'] = 'multiclass'  # hardcode for binary classification\n",
    "model_init_parameters['importance_type'] = 'gain'  # hardcode for gain\n",
    "model_init_parameters['learning_rate'] = 0.1\n",
    "model_init_parameters['num_iterations'] = 100\n",
    "model_init_parameters['num_leaves'] = 31\n",
    "model_init_parameters['bagging_fraction'] = 0.5\n",
    "model_init_parameters['bagging_freq'] = 1\n",
    "\n",
    "model_rf_m_classifier = lgbm.LGBMClassifier(**model_init_parameters).fit(X_class, y_class)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Binary Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init_parameters={}\n",
    "model_init_parameters['boosting'] = 'rf' \n",
    "model_init_parameters['objective'] = 'binary'  # hardcode for binary classification\n",
    "model_init_parameters['importance_type'] = 'gain'  # hardcode for gain\n",
    "model_init_parameters['learning_rate'] = 0.1\n",
    "model_init_parameters['num_iterations'] = 100\n",
    "model_init_parameters['num_leaves'] = 31\n",
    "model_init_parameters['bagging_fraction'] = 0.5\n",
    "model_init_parameters['bagging_freq'] = 1\n",
    "\n",
    "model_rf_b_classifier = lgbm.LGBMClassifier(**model_init_parameters).fit(X_class, y_class_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init_parameters={}\n",
    "model_init_parameters['boosting'] = 'rf' \n",
    "model_init_parameters['objective'] = 'regression'  # hardcode for binary classification\n",
    "model_init_parameters['importance_type'] = 'gain'  # hardcode for gain\n",
    "model_init_parameters['learning_rate'] = 0.1\n",
    "model_init_parameters['num_iterations'] = 100\n",
    "model_init_parameters['num_leaves'] = 31\n",
    "\n",
    "model_init_parameters['reg_alpha'] = 0.5\n",
    "model_init_parameters['reg_lambda'] = 0.5\n",
    "\n",
    "model_init_parameters['bagging_fraction'] = 0.5\n",
    "model_init_parameters['bagging_freq'] = 1\n",
    "\n",
    "model_rf_regression = lgbm.LGBMRegressor(**model_init_parameters).fit(X_regr, y_regr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Serialized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "datjson.to_json(model_logreg, 'model_logreg.json')\n",
    "datjson.to_json(model_linear, 'model_linear.json')\n",
    "datjson.to_json(model_elastic, 'model_elastic.json')\n",
    "datjson.to_json(model_ridge, 'model_ridge.json')\n",
    "datjson.to_json(model_lasso, 'model_lasso.json')\n",
    "\n",
    "datjson.to_json(model_kmeans, 'model_kmeans.json')\n",
    "datjson.to_json(model_dbscan, 'model_dbscan.json')\n",
    "\n",
    "datjson.to_json(model_lgbm_m_classifier, 'model_lgbm_m_classifier.json')\n",
    "datjson.to_json(model_lgbm_b_classifier, 'model_lgbm_b_classifier.json')\n",
    "\n",
    "datjson.to_json(model_rf_m_classifier, 'model_rf_m_classifier.json')\n",
    "datjson.to_json(model_rf_b_classifier, 'model_rf_b_classifier.json')\n",
    "\n",
    "datjson.to_json(model_rf_regression, 'model_rf_regression.json')\n",
    "datjson.to_json(model_lgbm_regression, 'model_lgbm_regression.json')\n",
    "\n",
    "datjson.to_json(model_iforest, 'model_iforest.json')\n",
    "\n",
    "datjson.to_json(encoder_label, 'encoder_label.json')\n",
    "datjson.to_json(encoder_onehot, 'encoder_onehot.json')\n",
    "\n",
    "datjson.to_json(scaler_min_max, 'scaler_min_max.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [[model_logreg, 'model_logreg.json', X_class],\n",
    "          [model_linear, 'model_linear.json', X_regr],\n",
    "          [model_elastic, 'model_elastic.json', X_regr],\n",
    "          [model_ridge, 'model_ridge.json', X_regr],\n",
    "          [model_lasso, 'model_lasso.json', X_regr],\n",
    "          [model_kmeans, 'model_kmeans.json', X_class],\n",
    "          [model_dbscan, 'model_dbscan.json', X_class],\n",
    "          [model_lgbm_m_classifier, 'model_lgbm_m_classifier.json', X_class],\n",
    "          [model_lgbm_b_classifier, 'model_lgbm_b_classifier.json', X_class],\n",
    "          [model_rf_m_classifier, 'model_rf_m_classifier.json', X_class],\n",
    "          [model_rf_b_classifier, 'model_rf_b_classifier.json', X_class],\n",
    "          [model_rf_regression, 'model_rf_regression.json', X_regr],\n",
    "          [model_lgbm_regression, 'model_lgbm_regression.json', X_regr],\n",
    "          [model_iforest, 'model_iforest.json', X_class],\n",
    "          [encoder_label, 'encoder_label.json', yy[['target']].categorize()],\n",
    "          [encoder_onehot, 'encoder_onehot.json', yy[['target']].categorize()],\n",
    "          [scaler_min_max, 'scaler_min_max.json', xx[['sepal length']]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model_logreg :  True\n",
      "model_linear :  True\n",
      "model_elastic :  True\n",
      "model_ridge :  True\n",
      "model_lasso :  True\n",
      "model_kmeans :  True\n",
      "model_dbscan :  True\n",
      "Finished loading model, total used 100 iterations\n",
      "model_lgbm_m_classifier :  True\n",
      "Finished loading model, total used 100 iterations\n",
      "model_lgbm_b_classifier :  True\n",
      "Finished loading model, total used 100 iterations\n",
      "model_rf_m_classifier :  True\n",
      "Finished loading model, total used 100 iterations\n",
      "model_rf_b_classifier :  True\n",
      "Finished loading model, total used 100 iterations\n",
      "model_rf_regression :  True\n",
      "Finished loading model, total used 100 iterations\n",
      "model_lgbm_regression :  True\n",
      "model_iforest :  True\n",
      "encoder_label :  True\n",
      "encoder_onehot :  True\n",
      "scaler_min_max :  True\n"
     ]
    }
   ],
   "source": [
    "for model in models:\n",
    "    deserialized_model = datjson.from_json(model[1])\n",
    "    print(model[1].split('.')[0],\": \", validate_model(model[0], deserialized_model, model[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}