examples/Pipelines-with-Gordo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Build pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.preprocessing import MinMaxScaler, QuantileTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import IsolationForest\n",
    "\n",
    "# Some pipeline we decide is 'the best' for our problem.\n",
    "pipe = Pipeline([\n",
    "    ('pca', PCA(n_components=10)),\n",
    "    ('feature_union', FeatureUnion([\n",
    "        ('feature_union_pipe1', Pipeline([\n",
    "            ('min_max', MinMaxScaler()),\n",
    "            ('pca', PCA(n_components=2))\n",
    "        ])),\n",
    "        ('feature_union_pipe2', Pipeline([\n",
    "            ('qt', QuantileTransformer(output_distribution='uniform')),\n",
    "        ]))\n",
    "    ])),\n",
    "    ('iso', IsolationForest(behaviour='new', contamination='auto'))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write this model architecture to a `Gordo Model Definition`\n",
    "\n",
    "### This a valid mapping to the Gordo config `model` key."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------\n",
      "sklearn.pipeline.Pipeline:\n",
      "  memory: null\n",
      "  steps:\n",
      "  - sklearn.decomposition.pca.PCA: {copy: true, iterated_power: auto, n_components: 10,\n",
      "      random_state: null, svd_solver: auto, tol: 0.0, whiten: false}\n",
      "  - sklearn.pipeline.FeatureUnion:\n",
      "      n_jobs: null\n",
      "      transformer_list:\n",
      "      - sklearn.pipeline.Pipeline:\n",
      "          memory: null\n",
      "          steps:\n",
      "          - sklearn.preprocessing.data.MinMaxScaler:\n",
      "              copy: true\n",
      "              feature_range: !!python/tuple [0, 1]\n",
      "          - sklearn.decomposition.pca.PCA: {copy: true, iterated_power: auto, n_components: 2,\n",
      "              random_state: null, svd_solver: auto, tol: 0.0, whiten: false}\n",
      "      - sklearn.pipeline.Pipeline:\n",
      "          memory: null\n",
      "          steps:\n",
      "          - sklearn.preprocessing.data.QuantileTransformer: {copy: true, ignore_implicit_zeros: false,\n",
      "              n_quantiles: 1000, output_distribution: uniform, random_state: null,\n",
      "              subsample: 100000}\n",
      "      transformer_weights: null\n",
      "  - sklearn.ensemble.iforest.IsolationForest: {behaviour: new, bootstrap: false, contamination: auto,\n",
      "      max_features: 1.0, max_samples: auto, n_estimators: 100, n_jobs: null, random_state: null,\n",
      "      verbose: 0}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from gordo.serializer import pipeline_into_definition\n",
    "\n",
    "yaml_definition_of_pipeline = yaml.dump(pipeline_into_definition(pipe))\n",
    "print('-' * 30)\n",
    "print(yaml_definition_of_pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load definition back into a pipeline\n",
    "\n",
    "### You probably won't need to do this, but it's how we get a replica of your defined model in Gordo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('step_0',\n",
      "  PCA(copy=True, iterated_power='auto', n_components=10, random_state=None,\n",
      "  svd_solver='auto', tol=0.0, whiten=False)),\n",
      " ('step_1',\n",
      "  FeatureUnion(n_jobs=None,\n",
      "       transformer_list=[('step_0', Pipeline(memory=None,\n",
      "     steps=[('step_0', MinMaxScaler(copy=True, feature_range=(0, 1))), ('step_1', PCA(copy=True, iterated_power='auto', n_components=2, random_state=None,\n",
      "  svd_solver='auto', tol=0.0, whiten=False))])), ('step_1', Pipeline(memory=None,\n",
      "     steps=[('step_0', QuantileTransformer(copy=True, ignore_implicit_zeros=False, n_quantiles=1000,\n",
      "          output_distribution='uniform', random_state=None,\n",
      "          subsample=100000))]))],\n",
      "       transformer_weights=None)),\n",
      " ('step_2',\n",
      "  IsolationForest(behaviour='new', bootstrap=False, contamination='auto',\n",
      "        max_features=1.0, max_samples='auto', n_estimators=100,\n",
      "        n_jobs=None, random_state=None, verbose=0))]\n"
     ]
    }
   ],
   "source": [
    "from gordo.serializer import pipeline_from_definition\n",
    "\n",
    "\n",
    "pipe = pipeline_from_definition(yaml.load(yaml_definition_of_pipeline))\n",
    "pprint(pipe.steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let us train the pipelilne."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000, 20)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "X = np.random.random(int(1e5)).reshape(-1, 20)\n",
    "y = X.copy()\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('step_0', PCA(copy=True, iterated_power='auto', n_components=10, random_state=None,\n",
       "  svd_solver='auto', tol=0.0, whiten=False)), ('step_1', FeatureUnion(n_jobs=None,\n",
       "       transformer_list=[('step_0', Pipeline(memory=None,\n",
       "     steps=[('step_0', MinMaxScaler(copy=True, feature_range=(0, 1)...ures=1.0, max_samples='auto', n_estimators=100,\n",
       "        n_jobs=None, random_state=None, verbose=0))])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict as normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5000,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_anomolies = pipe.predict(X)\n",
    "predicted_anomolies.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now we want to serialize it, for some reason."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x1f\\x8b\\x08\\x00\\xbe\\x87Q\\\\\\x02\\xff\\xec\\xbcux\\x1c\\xdf\\x92\\xa6Yb'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gordo.serializer import dumps, loads\n",
    "\n",
    "serialized_pipe_bytes = dumps(pipe)\n",
    "serialized_pipe_bytes[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load it back, ensuring the state is kept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe_clone = loads(serialized_pipe_bytes)\n",
    "predictions = pipe_clone.predict(X)\n",
    "assert np.allclose(predicted_anomolies, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optionally, you can save it to a directory "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tempfile import TemporaryDirectory\n",
    "from gordo.serializer import dump, load\n",
    "\n",
    "with TemporaryDirectory() as tmp:\n",
    "    \n",
    "    # Dump pipe to directory\n",
    "    dump(pipe, tmp)\n",
    "    \n",
    "    # Load it back\n",
    "    pipe_clone = load(tmp)\n",
    "    \n",
    "    assert np.allclose(pipe_clone.predict(X), predicted_anomolies)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}