examples/Gordo-Workflow-High-Level.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Gordo from only a config file\n",
    "\n",
    "This is a higher level example of how gordo works.\n",
    "\n",
    "Train a model from a config file.\n",
    "\n",
    "---\n",
    "** Some slight difference in how it _actually_ works, in that we normally use some resources from `gordo-infrastructure` for parsing the config that we don't have access to here. Yet.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tempfile\n",
    "import yaml\n",
    "from pprint import pprint\n",
    "from dateutil.parser import isoparse\n",
    "\n",
    "from gordo import serializer\n",
    "from gordo.builder import local_build\n",
    "from gordo.machine.dataset.data_provider.providers import DataLakeProvider"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define some config file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = \\\n",
    "\"\"\"\n",
    "machines:\n",
    "  - dataset:\n",
    "      tags:\n",
    "        - SOME-TAG1\n",
    "        - SOME-TAG2\n",
    "      target_tag_list:\n",
    "        - SOME-TAG3\n",
    "        - SOME-TAG4\n",
    "      train_end_date: '2019-03-01T00:00:00+00:00'\n",
    "      train_start_date: '2019-01-01T00:00:00+00:00'\n",
    "      asset: asgb\n",
    "      data_provider:\n",
    "        type: RandomDataProvider\n",
    "    metadata:\n",
    "      information: Some sweet information about the model\n",
    "    model:\n",
    "      gordo.machine.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
    "        base_estimator:\n",
    "          sklearn.pipeline.Pipeline:\n",
    "            steps:\n",
    "            - sklearn.decomposition.pca.PCA\n",
    "            - sklearn.multioutput.MultiOutputRegressor:\n",
    "                estimator: sklearn.linear_model.base.LinearRegression\n",
    "    name: crazy-sweet-name\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build model from data and model configs\n",
    "\n",
    "`gordo.builder.local_build` is a generator for building each machine in your string config, much in the same actual deployment on k8s works. \n",
    "\n",
    "Here we'll simply cast it as a list and get the first element, which is a `Tuple[BaseEstimator, dict]` of your model and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe, metadata = next(local_build(config))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### The trained model/pipeline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DiffBasedAnomalyDetector(base_estimator=Pipeline(memory=None,\n",
       "                                                 steps=[('step_0',\n",
       "                                                         PCA(copy=True,\n",
       "                                                             iterated_power='auto',\n",
       "                                                             n_components=None,\n",
       "                                                             random_state=None,\n",
       "                                                             svd_solver='auto',\n",
       "                                                             tol=0.0,\n",
       "                                                             whiten=False)),\n",
       "                                                        ('step_1',\n",
       "                                                         MultiOutputRegressor(estimator=LinearRegression(copy_X=True,\n",
       "                                                                                                         fit_intercept=True,\n",
       "                                                                                                         n_jobs=None,\n",
       "                                                                                                         normalize=False),\n",
       "                                                                              n_jobs=None))],\n",
       "                                                 verbose=False),\n",
       "                         scaler=RobustScaler(copy=True,\n",
       "                                             quantile_range=(25.0, 75.0),\n",
       "                                             with_centering=True,\n",
       "                                             with_scaling=True))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metadata from the model and build process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset:\n",
      "  asset: asgb\n",
      "  data_provider:\n",
      "    type: RandomDataProvider\n",
      "  filter: ''\n",
      "  resolution: 10T\n",
      "  row_filter_buffer_size: 0\n",
      "  tag_list:\n",
      "  - !!python/object/new:gordo.machine.dataset.sensor_tag.SensorTag\n",
      "    - SOME-TAG1\n",
      "    - asgb\n",
      "  - !!python/object/new:gordo.machine.dataset.sensor_tag.SensorTag\n",
      "    - SOME-TAG2\n",
      "    - asgb\n",
      "  target_tag_list:\n",
      "  - !!python/object/new:gordo.machine.dataset.sensor_tag.SensorTag\n",
      "    - SOME-TAG3\n",
      "    - asgb\n",
      "  - !!python/object/new:gordo.machine.dataset.sensor_tag.SensorTag\n",
      "    - SOME-TAG4\n",
      "    - asgb\n",
      "  train_end_date: 2019-03-01 00:00:00+00:00\n",
      "  train_start_date: 2019-01-01 00:00:00+00:00\n",
      "model:\n",
      "  base_estimator: \"Pipeline(memory=None,\\n         steps=[('step_0',\\n           \\\n",
      "    \\      PCA(copy=True, iterated_power='auto', n_components=None,\\n            \\\n",
      "    \\         random_state=None, svd_solver='auto', tol=0.0,\\n                   \\\n",
      "    \\  whiten=False)),\\n                ('step_1',\\n                 MultiOutputRegressor(estimator=LinearRegression(copy_X=True,\\n\\\n",
      "    \\                                                                 fit_intercept=True,\\n\\\n",
      "    \\                                                                 n_jobs=None,\\n\\\n",
      "    \\                                                                 normalize=False),\\n\\\n",
      "    \\                                      n_jobs=None))],\\n         verbose=False)\"\n",
      "  cross-validation:\n",
      "    cv-duration-sec: 0.09302020072937012\n",
      "    scores:\n",
      "      explained-variance-score:\n",
      "        fold-1: -0.01003142929062839\n",
      "        fold-2: -0.023048340085956576\n",
      "        fold-3: 0.004580463673101887\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - &id001 !!python/object/apply:numpy.dtype\n",
      "          args:\n",
      "          - f8\n",
      "          - 0\n",
      "          - 1\n",
      "          state: !!python/tuple\n",
      "          - 3\n",
      "          - <\n",
      "          - null\n",
      "          - null\n",
      "          - null\n",
      "          - -1\n",
      "          - -1\n",
      "          - 0\n",
      "        - !!binary |\n",
      "          wOXS2vbCcj8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          YA+MWp10g78=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          oMeV6/uZl78=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          ali5rvEchz8=\n",
      "      explained-variance-score-SOME-TAG3:\n",
      "        fold-1: 0.005345798219553499\n",
      "        fold-2: -0.030705948717322018\n",
      "        fold-3: -0.00994984963588319\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          AEtiyHnldT8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          VUIfqt4aiL8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          QFsiVmFxn78=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          kz5oovNBjj8=\n",
      "      explained-variance-score-SOME-TAG4:\n",
      "        fold-1: -0.02540865680081028\n",
      "        fold-2: -0.015390731454591133\n",
      "        fold-3: 0.019110776982086963\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          4I2OiMaRkz8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          1bjxFbicfb8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          gKT6F7oEmr8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          F8CNGt2Gkz8=\n",
      "      mean-absolute-error:\n",
      "        fold-1: 0.20583648036106478\n",
      "        fold-2: 0.24596998580111584\n",
      "        fold-3: 0.2630367456003708\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          yP3/EpjV0D8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          yPG2fv5/zj8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          Dr28i9lYyj8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          EatRHEqNmD8=\n",
      "      mean-absolute-error-SOME-TAG3:\n",
      "        fold-1: 0.19954794958775873\n",
      "        fold-2: 0.23592444772721596\n",
      "        fold-3: 0.2693421853842126\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          nCJqAec80T8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          aflnanQSzj8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          xky7hsmKyT8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          z54o15ovnT8=\n",
      "      mean-absolute-error-SOME-TAG4:\n",
      "        fold-1: 0.21212501113437085\n",
      "        fold-2: 0.25601552387501575\n",
      "        fold-3: 0.25673130581652903\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          9NiVJElu0D8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          KOoFk4jtzj8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          Vy2+kOkmyz8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          LbVORpJclT8=\n",
      "      mean-squared-error:\n",
      "        fold-1: 0.06401748946406775\n",
      "        fold-2: 0.07656939409345276\n",
      "        fold-3: 0.09058068975177724\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          qAwizEswtz8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          fBwWxe65sz8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          I8WeP3NjsD8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          JO2qPHA4hj8=\n",
      "      mean-squared-error-SOME-TAG3:\n",
      "        fold-1: 0.062121147362412706\n",
      "        fold-2: 0.06955663788601531\n",
      "        fold-3: 0.09405995609684242\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          DJRMM1AUuD8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          B6zZ8lBDsz8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          sYSf0FfOrz8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          g3YrUibyiz8=\n",
      "      mean-squared-error-SOME-TAG4:\n",
      "        fold-1: 0.0659138315657228\n",
      "        fold-2: 0.0835821503008902\n",
      "        fold-3: 0.08710142340671208\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          RYX3ZEdMtj8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          8IxSl4wwtD8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          7cftlrrfsD8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          6ENo1GD8gj8=\n",
      "      r2-score:\n",
      "        fold-1: -0.04759283408320136\n",
      "        fold-2: -0.051744726889909165\n",
      "        fold-3: -0.08382530453890136\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          cJbSgxZeqL8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          YHzP+IFCr78=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          6GKSPZN1tb8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          77qmvz6UkD8=\n",
      "      r2-score-SOME-TAG3:\n",
      "        fold-1: -0.06093981527814796\n",
      "        fold-2: -0.07763125614765176\n",
      "        fold-3: -0.009976559155124187\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          gAhiGpduhL8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          y91+HyVaqb8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          QDNNWqTfs78=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          Hq+4bbR3nT8=\n",
      "      r2-score-SOME-TAG4:\n",
      "        fold-1: -0.03424585288825477\n",
      "        fold-2: -0.02585819763216657\n",
      "        fold-3: -0.15767404992267853\n",
      "        fold-max: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          gJanRJJ6mr8=\n",
      "        fold-mean: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          ew0QaW+Vsr8=\n",
      "        fold-min: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          YELsy6kuxL8=\n",
      "        fold-std: !!python/object/apply:numpy.core.multiarray.scalar\n",
      "        - *id001\n",
      "        - !!binary |\n",
      "          BqxVhUParj8=\n",
      "  data-query-duration-sec: 0.0615231990814209\n",
      "  model-builder-version: 0.37.0rc2.dev8+ga427396.d20191021\n",
      "  model-config:\n",
      "    gordo.machine.model.anomaly.diff.DiffBasedAnomalyDetector:\n",
      "      base_estimator:\n",
      "        sklearn.pipeline.Pipeline:\n",
      "          steps:\n",
      "          - sklearn.decomposition.pca.PCA\n",
      "          - sklearn.multioutput.MultiOutputRegressor:\n",
      "              estimator: sklearn.linear_model.base.LinearRegression\n",
      "  model-creation-date: '2019-10-21 12:56:18.169609+02:00'\n",
      "  model-offset: 0\n",
      "  model-training-duration-sec: 0.005204916000366211\n",
      "  scaler: \"RobustScaler(copy=True, quantile_range=(25.0, 75.0), with_centering=True,\\n\\\n",
      "    \\             with_scaling=True)\"\n",
      "name: crazy-sweet-name\n",
      "user-defined:\n",
      "  global-metadata: {}\n",
      "  machine-metadata:\n",
      "    information: Some sweet information about the model\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(yaml.dump(metadata))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}