pytorch_sagemaker_tunner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --upgrade sagemaker awscli boto3 pandas Pillow==7.1.2 -i https://pypi.tuna.tsinghua.edu.cn/simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip uninstall -y tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from sagemaker.tuner import IntegerParameter, CategoricalParameter, ContinuousParameter, HyperparameterTuner\n",
    "\n",
    "from code.dataloader import get_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "dataroot = './data'\n",
    "\n",
    "# down load dataset if not exist\n",
    "trainset = torchvision.datasets.CIFAR10(root=dataroot, train=True, download=True)\n",
    "testset = torchvision.datasets.CIFAR10(root=dataroot, train=False, download=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* upload dataset to s3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updload to s3://flow-sagemaker-bucket/data/cifar-10-batches-py\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from sagemaker.s3 import S3Uploader as s3up\n",
    "\n",
    "# S3 bucket for saving code and model artifacts.\n",
    "bucket = 'flow-sagemaker-bucket'\n",
    "\n",
    "s3_data_location = s3up.upload(os.path.join(dataroot, \"cifar-10-batches-py\"), f\"s3://{bucket}/data/cifar-10-batches-py\")\n",
    "print('Updload to {0}'.format(s3_data_location))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A. Hyper paramter tunning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use IAM role arn:aws:iam::843456404290:role/service-role/AmazonSageMaker-ExecutionRole-20190516T112244\n"
     ]
    }
   ],
   "source": [
    "from sagemaker import get_execution_role\n",
    "\n",
    "# IAM execution role that gives SageMaker access to resources in your AWS account.\n",
    "# We can use the SageMaker Python SDK to get the role from our notebook environment. \n",
    "role = get_execution_role()\n",
    "print('Use IAM role {0}'.format(role))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.pytorch import PyTorch\n",
    "\n",
    "# Location to save your custom code in tar.gz format.\n",
    "s3_custom_code_upload_location = f's3://{bucket}/code'\n",
    "\n",
    "# Location where results of model training are saved.\n",
    "s3_model_artifacts_location = f's3://{bucket}/models/'\n",
    "\n",
    "estimator = PyTorch(role=role,\n",
    "                    entry_point='train.py',\n",
    "                    source_dir='./code',\n",
    "                    output_path=s3_model_artifacts_location,\n",
    "                    code_location=s3_custom_code_upload_location,\n",
    "                    instance_count=1,\n",
    "                    instance_type='ml.p3.2xlarge',\n",
    "                    use_spot_instances=True,\n",
    "                    max_wait=86400,\n",
    "                    framework_version='1.5.0',\n",
    "                    py_version='py3',\n",
    "#                     hyperparameters=hps,\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hyperparameter_ranges = {'lr': ContinuousParameter(0.001, 0.1),'batch-size': CategoricalParameter([32,64,128,256,512])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "objective_metric_name = 'average test loss'\n",
    "objective_type = 'Minimize'\n",
    "# use Regex to extract loss from log\n",
    "metric_definitions = [{'Name': 'average test loss',\n",
    "                       'Regex': 'Test set: Average loss: ([0-9\\\\.]+)'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max_jobs will sample hyperparameter_ranges into specify job number\n",
    "tuner = HyperparameterTuner(estimator,\n",
    "                            objective_metric_name,\n",
    "                            hyperparameter_ranges,\n",
    "                            metric_definitions,\n",
    "                            max_jobs=9,\n",
    "                            max_parallel_jobs=2,\n",
    "                            objective_type=objective_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......................................................................................................................................................................................................................................................................................................................................................................................!\n"
     ]
    }
   ],
   "source": [
    "tuner.fit({'training': s3_data_location})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B. Track hyperparameter tuning job progress\n",
    "* After you launch a tuning job, you can see its progress by calling describe_tuning_job API.\n",
    "* The output from describe-tuning-job is a JSON object that contains information about the current state of the tuning job(see section 2 below)\n",
    "* You can call list_training_jobs_for_tuning_job to see a detailed list of the training jobs that the tuning job launched(see section 1 below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import boto3\n",
    "import sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "region = boto3.Session().region_name\n",
    "sage_client = boto3.Session().client('sagemaker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## You must have already run a hyperparameter tuning job to analyze it here.\n",
    "## The Hyperparameter tuning jobs you have run are listed in the Training section on your SageMaker dashboard.\n",
    "## Copy the name of a completed job you want to analyze from that list.\n",
    "## For example: tuning_job_name = 'mxnet-training-201007-0054'.\n",
    "tuning_job_name = 'pytorch-training-201230-1148'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Track all tunning jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2020-12-30 12:19:22 Starting - Preparing the instances for training\n",
      "2020-12-30 12:19:22 Downloading - Downloading input data\n",
      "2020-12-30 12:19:22 Training - Training image download completed. Training in progress.\n",
      "2020-12-30 12:19:22 Uploading - Uploading generated training model\n",
      "2020-12-30 12:19:22 Completed - Training job completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<sagemaker.estimator.Estimator at 0x7fbeb6b4f438>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "all_jobs_info = sage_client.list_training_jobs_for_hyper_parameter_tuning_job(HyperParameterTuningJobName=tuning_job_name)\n",
    "# track first tunning\n",
    "sub_jobs_info = all_jobs_info['TrainingJobSummaries'][0]\n",
    "\n",
    "attached_estimator = Estimator.attach(sub_jobs_info['TrainingJobName'])\n",
    "attached_estimator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Tracking tunning state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 training jobs have completed\n",
      "is_minimize:True, objective nameaverage test loss\n"
     ]
    }
   ],
   "source": [
    "# run this cell to check current status of hyperparameter tuning job\n",
    "tuning_job_result = sage_client.describe_hyper_parameter_tuning_job(HyperParameterTuningJobName=tuning_job_name)\n",
    "\n",
    "status = tuning_job_result['HyperParameterTuningJobStatus']\n",
    "if status != 'Completed':\n",
    "    print('Reminder: the tuning job has not been completed.')\n",
    "    \n",
    "job_count = tuning_job_result['TrainingJobStatusCounters']['Completed']\n",
    "print(\"%d training jobs have completed\" % job_count)\n",
    "    \n",
    "is_minimize = (tuning_job_result['HyperParameterTuningJobConfig']['HyperParameterTuningJobObjective']['Type'] != 'Maximize')\n",
    "objective_name = tuning_job_result['HyperParameterTuningJobConfig']['HyperParameterTuningJobObjective']['MetricName']\n",
    "print('is_minimize:{0}, objective name{1}'.format(is_minimize, objective_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best model found so far:\n",
      "{'CreationTime': datetime.datetime(2020, 12, 30, 11, 54, 37, tzinfo=tzlocal()),\n",
      " 'FinalHyperParameterTuningJobObjectiveMetric': {'MetricName': 'average test '\n",
      "                                                               'loss',\n",
      "                                                 'Value': 2.2894999980926514},\n",
      " 'ObjectiveStatus': 'Succeeded',\n",
      " 'TrainingEndTime': datetime.datetime(2020, 12, 30, 12, 0, 4, tzinfo=tzlocal()),\n",
      " 'TrainingJobArn': 'arn:aws:sagemaker:ap-northeast-1:843456404290:training-job/pytorch-training-201230-1148-003-7de4e459',\n",
      " 'TrainingJobName': 'pytorch-training-201230-1148-003-7de4e459',\n",
      " 'TrainingJobStatus': 'Completed',\n",
      " 'TrainingStartTime': datetime.datetime(2020, 12, 30, 11, 57, 3, tzinfo=tzlocal()),\n",
      " 'TunedHyperParameters': {'batch-size': '\"256\"', 'lr': '0.001297084130092771'}}\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "if tuning_job_result.get('BestTrainingJob',None):\n",
    "    print(\"Best model found so far:\")\n",
    "    pprint(tuning_job_result['BestTrainingJob'])\n",
    "else:\n",
    "    print(\"No training jobs have reported results yet.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch all results as DataFrame\n",
    "* We can list hyperparameters and objective metrics of all training jobs and pick up the training job with the best objective metric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training jobs with valid objective: 9\n",
      "{'lowest': 2.2894999980926514, 'highest': 2.3096001148223877}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batch-size</th>\n",
       "      <th>lr</th>\n",
       "      <th>TrainingJobName</th>\n",
       "      <th>TrainingJobStatus</th>\n",
       "      <th>FinalObjectiveValue</th>\n",
       "      <th>TrainingStartTime</th>\n",
       "      <th>TrainingEndTime</th>\n",
       "      <th>TrainingElapsedTimeSeconds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"256\"</td>\n",
       "      <td>0.001297</td>\n",
       "      <td>pytorch-training-201230-1148-003-7de4e459</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.2895</td>\n",
       "      <td>2020-12-30 11:57:03+00:00</td>\n",
       "      <td>2020-12-30 12:00:04+00:00</td>\n",
       "      <td>181.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>\"256\"</td>\n",
       "      <td>0.001713</td>\n",
       "      <td>pytorch-training-201230-1148-002-b13ed748</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.2955</td>\n",
       "      <td>2020-12-30 11:51:15+00:00</td>\n",
       "      <td>2020-12-30 11:54:16+00:00</td>\n",
       "      <td>181.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"64\"</td>\n",
       "      <td>0.014973</td>\n",
       "      <td>pytorch-training-201230-1148-007-42d5e596</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.2989</td>\n",
       "      <td>2020-12-30 12:09:34+00:00</td>\n",
       "      <td>2020-12-30 12:12:36+00:00</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\"256\"</td>\n",
       "      <td>0.004586</td>\n",
       "      <td>pytorch-training-201230-1148-001-d7e279dc</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.3046</td>\n",
       "      <td>2020-12-30 11:50:56+00:00</td>\n",
       "      <td>2020-12-30 11:53:52+00:00</td>\n",
       "      <td>176.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"128\"</td>\n",
       "      <td>0.099080</td>\n",
       "      <td>pytorch-training-201230-1148-009-b457940d</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.3059</td>\n",
       "      <td>2020-12-30 12:16:16+00:00</td>\n",
       "      <td>2020-12-30 12:19:22+00:00</td>\n",
       "      <td>186.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"256\"</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>pytorch-training-201230-1148-005-7c96a996</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.3089</td>\n",
       "      <td>2020-12-30 12:04:21+00:00</td>\n",
       "      <td>2020-12-30 12:07:17+00:00</td>\n",
       "      <td>176.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"64\"</td>\n",
       "      <td>0.014299</td>\n",
       "      <td>pytorch-training-201230-1148-008-d7878ef8</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.3091</td>\n",
       "      <td>2020-12-30 12:10:08+00:00</td>\n",
       "      <td>2020-12-30 12:13:10+00:00</td>\n",
       "      <td>182.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"256\"</td>\n",
       "      <td>0.001010</td>\n",
       "      <td>pytorch-training-201230-1148-006-8f209098</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.3095</td>\n",
       "      <td>2020-12-30 12:03:16+00:00</td>\n",
       "      <td>2020-12-30 12:06:11+00:00</td>\n",
       "      <td>175.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>\"512\"</td>\n",
       "      <td>0.001254</td>\n",
       "      <td>pytorch-training-201230-1148-004-213737ce</td>\n",
       "      <td>Completed</td>\n",
       "      <td>2.3096</td>\n",
       "      <td>2020-12-30 11:57:29+00:00</td>\n",
       "      <td>2020-12-30 12:00:15+00:00</td>\n",
       "      <td>166.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  batch-size        lr                            TrainingJobName  \\\n",
       "6  \"256\"      0.001297  pytorch-training-201230-1148-003-7de4e459   \n",
       "7  \"256\"      0.001713  pytorch-training-201230-1148-002-b13ed748   \n",
       "2  \"64\"       0.014973  pytorch-training-201230-1148-007-42d5e596   \n",
       "8  \"256\"      0.004586  pytorch-training-201230-1148-001-d7e279dc   \n",
       "0  \"128\"      0.099080  pytorch-training-201230-1148-009-b457940d   \n",
       "4  \"256\"      0.001000  pytorch-training-201230-1148-005-7c96a996   \n",
       "1  \"64\"       0.014299  pytorch-training-201230-1148-008-d7878ef8   \n",
       "3  \"256\"      0.001010  pytorch-training-201230-1148-006-8f209098   \n",
       "5  \"512\"      0.001254  pytorch-training-201230-1148-004-213737ce   \n",
       "\n",
       "  TrainingJobStatus  FinalObjectiveValue         TrainingStartTime  \\\n",
       "6  Completed         2.2895              2020-12-30 11:57:03+00:00   \n",
       "7  Completed         2.2955              2020-12-30 11:51:15+00:00   \n",
       "2  Completed         2.2989              2020-12-30 12:09:34+00:00   \n",
       "8  Completed         2.3046              2020-12-30 11:50:56+00:00   \n",
       "0  Completed         2.3059              2020-12-30 12:16:16+00:00   \n",
       "4  Completed         2.3089              2020-12-30 12:04:21+00:00   \n",
       "1  Completed         2.3091              2020-12-30 12:10:08+00:00   \n",
       "3  Completed         2.3095              2020-12-30 12:03:16+00:00   \n",
       "5  Completed         2.3096              2020-12-30 11:57:29+00:00   \n",
       "\n",
       "            TrainingEndTime  TrainingElapsedTimeSeconds  \n",
       "6 2020-12-30 12:00:04+00:00  181.0                       \n",
       "7 2020-12-30 11:54:16+00:00  181.0                       \n",
       "2 2020-12-30 12:12:36+00:00  182.0                       \n",
       "8 2020-12-30 11:53:52+00:00  176.0                       \n",
       "0 2020-12-30 12:19:22+00:00  186.0                       \n",
       "4 2020-12-30 12:07:17+00:00  176.0                       \n",
       "1 2020-12-30 12:13:10+00:00  182.0                       \n",
       "3 2020-12-30 12:06:11+00:00  175.0                       \n",
       "5 2020-12-30 12:00:15+00:00  166.0                       "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "tuner_info = sagemaker.HyperparameterTuningJobAnalytics(tuning_job_name)\n",
    "\n",
    "full_df = tuner_info.dataframe()\n",
    "\n",
    "if len(full_df) > 0:\n",
    "    df = full_df[full_df['FinalObjectiveValue'] > -float('inf')]\n",
    "    if len(df) > 0:\n",
    "        df = df.sort_values('FinalObjectiveValue', ascending=is_minimize)\n",
    "        print(\"Number of training jobs with valid objective: %d\" % len(df))\n",
    "        print({\"lowest\":min(df['FinalObjectiveValue']),\"highest\": max(df['FinalObjectiveValue'])})\n",
    "        pd.set_option('display.max_colwidth', -1)  # Don't truncate TrainingJobName        \n",
    "    else:\n",
    "        print(\"No training jobs have reported valid results yet.\")\n",
    "        \n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## See TuningJob results vs time\n",
    "Next we will show how the objective metric changes over time, as the tuning job progresses.\n",
    "For Bayesian strategy, you should expect to see a general trend towards better results, but this progress will not be steady as the algorithm needs to balance exploration of new areas of parameter space against exploitation of known good areas. This can give you a sense of whether or not the number of training jobs is sufficient for the complexity of your search space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };var element = document.getElementById(\"1001\");\n",
       "  if (element == null) {\n",
       "    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  \n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
       "  var css_urls = [];\n",
       "  \n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "    \n",
       "    \n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      \n",
       "    for (var i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "    if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };var element = document.getElementById(\"1001\");\n  if (element == null) {\n    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n    return false;\n  }\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"61d4b00a-5d98-4e9d-8cfb-08b0f81d8ef3\" data-root-id=\"1003\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"da43467a-359a-4bf1-8a71-388b3e0606ad\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1012\",\"type\":\"DatetimeAxis\"}],\"center\":[{\"id\":\"1016\",\"type\":\"Grid\"},{\"id\":\"1021\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1017\",\"type\":\"LinearAxis\"}],\"plot_height\":400,\"plot_width\":900,\"renderers\":[{\"id\":\"1042\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1045\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1029\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1004\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1008\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1006\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1010\",\"type\":\"LinearScale\"}},\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"num_minor_ticks\":5,\"tickers\":[{\"id\":\"1050\",\"type\":\"AdaptiveTicker\"},{\"id\":\"1051\",\"type\":\"AdaptiveTicker\"},{\"id\":\"1052\",\"type\":\"AdaptiveTicker\"},{\"id\":\"1053\",\"type\":\"DaysTicker\"},{\"id\":\"1054\",\"type\":\"DaysTicker\"},{\"id\":\"1055\",\"type\":\"DaysTicker\"},{\"id\":\"1056\",\"type\":\"DaysTicker\"},{\"id\":\"1057\",\"type\":\"MonthsTicker\"},{\"id\":\"1058\",\"type\":\"MonthsTicker\"},{\"id\":\"1059\",\"type\":\"MonthsTicker\"},{\"id\":\"1060\",\"type\":\"MonthsTicker\"},{\"id\":\"1061\",\"type\":\"YearsTicker\"}]},\"id\":\"1013\",\"type\":\"DatetimeTicker\"},{\"attributes\":{\"mantissas\":[1,2,5],\"max_interval\":500.0,\"num_minor_ticks\":0},\"id\":\"1050\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"callback\":null},\"id\":\"1006\",\"type\":\"DataRange1d\"},{\"attributes\":{\"base\":60,\"mantissas\":[1,2,5,10,15,20,30],\"max_interval\":1800000.0,\"min_interval\":1000.0,\"num_minor_ticks\":0},\"id\":\"1051\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"callback\":null},\"id\":\"1004\",\"type\":\"DataRange1d\"},{\"attributes\":{\"data_source\":{\"id\":\"1038\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1040\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1041\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1043\",\"type\":\"CDSView\"}},\"id\":\"1042\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1038\",\"type\":\"ColumnDataSource\"}},\"id\":\"1043\",\"type\":\"CDSView\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"TrainingStartTime\"},\"y\":{\"field\":\"FinalObjectiveValue\"}},\"id\":\"1041\",\"type\":\"Circle\"},{\"attributes\":{\"base\":24,\"mantissas\":[1,2,4,6,8,12],\"max_interval\":43200000.0,\"min_interval\":3600000.0,\"num_minor_ticks\":0},\"id\":\"1052\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{},\"id\":\"1022\",\"type\":\"PanTool\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1045\",\"type\":\"Title\"},{\"attributes\":{\"formatter\":{\"id\":\"1049\",\"type\":\"DatetimeTickFormatter\"},\"ticker\":{\"id\":\"1013\",\"type\":\"DatetimeTicker\"}},\"id\":\"1012\",\"type\":\"DatetimeAxis\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1018\",\"type\":\"BasicTicker\"}},\"id\":\"1021\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"FinalObjectiveValue\",\"@FinalObjectiveValue\"],[\"TrainingJobName\",\"@TrainingJobName\"],[\"lr\",\"@{lr}\"],[\"batch-size\",\"@{batch-size}\"]]},\"id\":\"1002\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1010\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1023\",\"type\":\"CrosshairTool\"},{\"attributes\":{\"days\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]},\"id\":\"1053\",\"type\":\"DaysTicker\"},{\"attributes\":{\"days\":[1,4,7,10,13,16,19,22,25,28]},\"id\":\"1054\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"TrainingStartTime\"},\"y\":{\"field\":\"FinalObjectiveValue\"}},\"id\":\"1040\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1008\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1025\",\"type\":\"ZoomInTool\"},{\"attributes\":{\"days\":[1,8,15,22]},\"id\":\"1055\",\"type\":\"DaysTicker\"},{\"attributes\":{\"days\":[1,15]},\"id\":\"1056\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1026\",\"type\":\"ZoomOutTool\"},{\"attributes\":{\"months\":[0,1,2,3,4,5,6,7,8,9,10,11]},\"id\":\"1057\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1027\",\"type\":\"UndoTool\"},{\"attributes\":{\"ticker\":{\"id\":\"1013\",\"type\":\"DatetimeTicker\"}},\"id\":\"1016\",\"type\":\"Grid\"},{\"attributes\":{\"months\":[0,2,4,6,8,10]},\"id\":\"1058\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1028\",\"type\":\"ResetTool\"},{\"attributes\":{\"callback\":null,\"data\":{\"FinalObjectiveValue\":{\"__ndarray__\":\"AAAAYOVQAkAAAAAgL10CQAAAAKAlZAJAAAAAINJvAkAAAADAe3ICQAAAAKCgeAJAAAAAYAl5AkAAAAAg23kCQAAAAKAPegJA\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingElapsedTimeSeconds\":{\"__ndarray__\":\"AAAAAACgZkAAAAAAAKBmQAAAAAAAwGZAAAAAAAAAZkAAAAAAAEBnQAAAAAAAAGZAAAAAAADAZkAAAAAAAOBlQAAAAAAAwGRA\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingEndTime\":{\"__ndarray__\":\"AABaTzhrd0IAAGT6N2t3QgAA8gY5a3dCAACI9Ddrd0IAABFqOWt3QgCAELk4a3dCAAA/Dzlrd0IAgPOoOGt3QgCACVI4a3dC\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingJobName\":[\"pytorch-training-201230-1148-003-7de4e459\",\"pytorch-training-201230-1148-002-b13ed748\",\"pytorch-training-201230-1148-007-42d5e596\",\"pytorch-training-201230-1148-001-d7e279dc\",\"pytorch-training-201230-1148-009-b457940d\",\"pytorch-training-201230-1148-005-7c96a996\",\"pytorch-training-201230-1148-008-d7878ef8\",\"pytorch-training-201230-1148-006-8f209098\",\"pytorch-training-201230-1148-004-213737ce\"],\"TrainingJobStatus\":[\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\"],\"TrainingStartTime\":{\"__ndarray__\":\"AIApIzhrd0IAgDPON2t3QgAAg9o4a3dCAACQyTdrd0IAAKg8OWt3QgCAGI44a3dCAADQ4jhrd0IAADp+OGt3QgCAgik4a3dC\",\"dtype\":\"float64\",\"shape\":[9]},\"batch-size\":[\"\\\"256\\\"\",\"\\\"256\\\"\",\"\\\"64\\\"\",\"\\\"256\\\"\",\"\\\"128\\\"\",\"\\\"256\\\"\",\"\\\"64\\\"\",\"\\\"256\\\"\",\"\\\"512\\\"\"],\"index\":[6,7,2,8,0,4,1,3,5],\"lr\":{\"__ndarray__\":\"82fPel1AVT9YkPn4fRJcP+eufaghqo4/5iYPVILIcj9EJyqDTV25PybIjU29YlA/GuKBPtFIjT8HMECVeo1QP6olZnz5iVQ/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1063\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1062\",\"type\":\"UnionRenderers\"}},\"id\":\"1038\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"months\":[0,4,8]},\"id\":\"1059\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1002\",\"type\":\"HoverTool\"},{\"id\":\"1022\",\"type\":\"PanTool\"},{\"id\":\"1023\",\"type\":\"CrosshairTool\"},{\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"id\":\"1025\",\"type\":\"ZoomInTool\"},{\"id\":\"1026\",\"type\":\"ZoomOutTool\"},{\"id\":\"1027\",\"type\":\"UndoTool\"},{\"id\":\"1028\",\"type\":\"ResetTool\"}]},\"id\":\"1029\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1018\",\"type\":\"BasicTicker\"},{\"attributes\":{\"months\":[0,6]},\"id\":\"1060\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"formatter\":{\"id\":\"1047\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1018\",\"type\":\"BasicTicker\"}},\"id\":\"1017\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1047\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1061\",\"type\":\"YearsTicker\"},{\"attributes\":{},\"id\":\"1049\",\"type\":\"DatetimeTickFormatter\"},{\"attributes\":{},\"id\":\"1062\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1063\",\"type\":\"Selection\"}],\"root_ids\":[\"1003\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
       "  var render_items = [{\"docid\":\"da43467a-359a-4bf1-8a71-388b3e0606ad\",\"roots\":{\"1003\":\"61d4b00a-5d98-4e9d-8cfb-08b0f81d8ef3\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else {\n",
       "        attempts++;\n",
       "        if (attempts > 100) {\n",
       "          clearInterval(timer);\n",
       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        }\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1003"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import bokeh\n",
    "import bokeh.io\n",
    "bokeh.io.output_notebook()\n",
    "from bokeh.plotting import figure, show\n",
    "from bokeh.models import HoverTool\n",
    "\n",
    "class HoverHelper():\n",
    "\n",
    "    def __init__(self, tuning_analytics):\n",
    "        self.tuner = tuning_analytics\n",
    "\n",
    "    def hovertool(self):\n",
    "        tooltips = [\n",
    "            (\"FinalObjectiveValue\", \"@FinalObjectiveValue\"),\n",
    "            (\"TrainingJobName\", \"@TrainingJobName\"),\n",
    "        ]\n",
    "        for k in self.tuner.tuning_ranges.keys():\n",
    "            tooltips.append( (k, \"@{%s}\" % k) )\n",
    "\n",
    "        ht = HoverTool(tooltips=tooltips)\n",
    "        return ht\n",
    "\n",
    "    def tools(self, standard_tools='pan,crosshair,wheel_zoom,zoom_in,zoom_out,undo,reset'):\n",
    "        return [self.hovertool(), standard_tools]\n",
    "\n",
    "hover = HoverHelper(tuner_info)\n",
    "\n",
    "p = figure(plot_width=900, plot_height=400, tools=hover.tools(), x_axis_type='datetime')\n",
    "p.circle(source=df, x='TrainingStartTime', y='FinalObjectiveValue')\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze the correlation between objective metric and individual hyperparameters\n",
    "Now you have finished a tuning job, you may want to know the correlation between your objective metric and individual hyperparameters you've selected to tune. Having that insight will help you decide whether it makes sense to adjust search ranges for certain hyperparameters and start another tuning job. For example, if you see a positive trend between objective metric and a numerical hyperparameter, you probably want to set a higher tuning range for that hyperparameter in your next tuning job.\n",
    "\n",
    "The following cell draws a graph for each hyperparameter to show its correlation with your objective metric."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"769900e0-363c-4f71-aa95-68870f909480\" data-root-id=\"1271\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"c55983b1-780a-4a94-ad11-457acaea6dc7\":{\"roots\":{\"references\":[{\"attributes\":{\"children\":[{\"id\":\"1185\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"id\":\"1229\",\"subtype\":\"Figure\",\"type\":\"Plot\"}]},\"id\":\"1271\",\"type\":\"Column\"},{\"attributes\":{},\"id\":\"1241\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"lr\"},\"y\":{\"field\":\"FinalObjectiveValue\"}},\"id\":\"1224\",\"type\":\"Circle\"},{\"attributes\":{\"source\":{\"id\":\"1265\",\"type\":\"ColumnDataSource\"}},\"id\":\"1270\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1197\",\"type\":\"BasicTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"lr\"},\"y\":{\"field\":\"FinalObjectiveValue\"}},\"id\":\"1225\",\"type\":\"Circle\"},{\"attributes\":{\"ticker\":{\"id\":\"1241\",\"type\":\"CategoricalTicker\"}},\"id\":\"1243\",\"type\":\"Grid\"},{\"attributes\":{\"axis_label\":\"lr\",\"formatter\":{\"id\":\"1297\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1197\",\"type\":\"BasicTicker\"}},\"id\":\"1196\",\"type\":\"LinearAxis\"},{\"attributes\":{\"below\":[{\"id\":\"1240\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1243\",\"type\":\"Grid\"},{\"id\":\"1248\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1244\",\"type\":\"LinearAxis\"}],\"plot_height\":500,\"plot_width\":500,\"renderers\":[{\"id\":\"1269\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1230\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1256\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1232\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1236\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1234\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1238\",\"type\":\"LinearScale\"}},\"id\":\"1229\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"axis_label\":\"average test loss\",\"formatter\":{\"id\":\"1299\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1245\",\"type\":\"BasicTicker\"}},\"id\":\"1244\",\"type\":\"LinearAxis\"},{\"attributes\":{\"ticker\":{\"id\":\"1197\",\"type\":\"BasicTicker\"}},\"id\":\"1200\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1245\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1212\",\"type\":\"ResetTool\"},{\"attributes\":{\"callback\":null,\"data\":{\"FinalObjectiveValue\":{\"__ndarray__\":\"AAAAYOVQAkAAAAAgL10CQAAAAKAlZAJAAAAAINJvAkAAAADAe3ICQAAAAKCgeAJAAAAAYAl5AkAAAAAg23kCQAAAAKAPegJA\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingElapsedTimeSeconds\":{\"__ndarray__\":\"AAAAAACgZkAAAAAAAKBmQAAAAAAAwGZAAAAAAAAAZkAAAAAAAEBnQAAAAAAAAGZAAAAAAADAZkAAAAAAAOBlQAAAAAAAwGRA\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingEndTime\":{\"__ndarray__\":\"AABaTzhrd0IAAGT6N2t3QgAA8gY5a3dCAACI9Ddrd0IAABFqOWt3QgCAELk4a3dCAAA/Dzlrd0IAgPOoOGt3QgCACVI4a3dC\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingJobName\":[\"pytorch-training-201230-1148-003-7de4e459\",\"pytorch-training-201230-1148-002-b13ed748\",\"pytorch-training-201230-1148-007-42d5e596\",\"pytorch-training-201230-1148-001-d7e279dc\",\"pytorch-training-201230-1148-009-b457940d\",\"pytorch-training-201230-1148-005-7c96a996\",\"pytorch-training-201230-1148-008-d7878ef8\",\"pytorch-training-201230-1148-006-8f209098\",\"pytorch-training-201230-1148-004-213737ce\"],\"TrainingJobStatus\":[\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\"],\"TrainingStartTime\":{\"__ndarray__\":\"AIApIzhrd0IAgDPON2t3QgAAg9o4a3dCAACQyTdrd0IAAKg8OWt3QgCAGI44a3dCAADQ4jhrd0IAADp+OGt3QgCAgik4a3dC\",\"dtype\":\"float64\",\"shape\":[9]},\"batch-size\":[\"\\\"256\\\"\",\"\\\"256\\\"\",\"\\\"64\\\"\",\"\\\"256\\\"\",\"\\\"128\\\"\",\"\\\"256\\\"\",\"\\\"64\\\"\",\"\\\"256\\\"\",\"\\\"512\\\"\"],\"index\":[6,7,2,8,0,4,1,3,5],\"lr\":{\"__ndarray__\":\"82fPel1AVT9YkPn4fRJcP+eufaghqo4/5iYPVILIcj9EJyqDTV25PybIjU29YlA/GuKBPtFIjT8HMECVeo1QP6olZnz5iVQ/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1303\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1302\",\"type\":\"UnionRenderers\"}},\"id\":\"1222\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1202\",\"type\":\"BasicTicker\"}},\"id\":\"1205\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1297\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1299\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1245\",\"type\":\"BasicTicker\"}},\"id\":\"1248\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1210\",\"type\":\"ZoomOutTool\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"FinalObjectiveValue\",\"@FinalObjectiveValue\"],[\"TrainingJobName\",\"@TrainingJobName\"],[\"lr\",\"@{lr}\"],[\"batch-size\",\"@{batch-size}\"]]},\"id\":\"1184\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1202\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1301\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1209\",\"type\":\"ZoomInTool\"},{\"attributes\":{\"below\":[{\"id\":\"1196\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1200\",\"type\":\"Grid\"},{\"id\":\"1205\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1201\",\"type\":\"LinearAxis\"}],\"plot_height\":500,\"plot_width\":500,\"renderers\":[{\"id\":\"1226\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1186\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1213\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1188\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1192\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1190\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1194\",\"type\":\"LinearScale\"}},\"id\":\"1185\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"data\":{\"FinalObjectiveValue\":{\"__ndarray__\":\"AAAAYOVQAkAAAAAgL10CQAAAAKAlZAJAAAAAINJvAkAAAADAe3ICQAAAAKCgeAJAAAAAYAl5AkAAAAAg23kCQAAAAKAPegJA\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingElapsedTimeSeconds\":{\"__ndarray__\":\"AAAAAACgZkAAAAAAAKBmQAAAAAAAwGZAAAAAAAAAZkAAAAAAAEBnQAAAAAAAAGZAAAAAAADAZkAAAAAAAOBlQAAAAAAAwGRA\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingEndTime\":{\"__ndarray__\":\"AABaTzhrd0IAAGT6N2t3QgAA8gY5a3dCAACI9Ddrd0IAABFqOWt3QgCAELk4a3dCAAA/Dzlrd0IAgPOoOGt3QgCACVI4a3dC\",\"dtype\":\"float64\",\"shape\":[9]},\"TrainingJobName\":[\"pytorch-training-201230-1148-003-7de4e459\",\"pytorch-training-201230-1148-002-b13ed748\",\"pytorch-training-201230-1148-007-42d5e596\",\"pytorch-training-201230-1148-001-d7e279dc\",\"pytorch-training-201230-1148-009-b457940d\",\"pytorch-training-201230-1148-005-7c96a996\",\"pytorch-training-201230-1148-008-d7878ef8\",\"pytorch-training-201230-1148-006-8f209098\",\"pytorch-training-201230-1148-004-213737ce\"],\"TrainingJobStatus\":[\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\",\"Completed\"],\"TrainingStartTime\":{\"__ndarray__\":\"AIApIzhrd0IAgDPON2t3QgAAg9o4a3dCAACQyTdrd0IAAKg8OWt3QgCAGI44a3dCAADQ4jhrd0IAADp+OGt3QgCAgik4a3dC\",\"dtype\":\"float64\",\"shape\":[9]},\"batch-size\":[\"\\\"256\\\"\",\"\\\"256\\\"\",\"\\\"64\\\"\",\"\\\"256\\\"\",\"\\\"128\\\"\",\"\\\"256\\\"\",\"\\\"64\\\"\",\"\\\"256\\\"\",\"\\\"512\\\"\"],\"index\":[6,7,2,8,0,4,1,3,5],\"lr\":{\"__ndarray__\":\"82fPel1AVT9YkPn4fRJcP+eufaghqo4/5iYPVILIcj9EJyqDTV25PybIjU29YlA/GuKBPtFIjT8HMECVeo1QP6olZnz5iVQ/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1305\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1304\",\"type\":\"UnionRenderers\"}},\"id\":\"1265\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1303\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1304\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"text\":\"Objective vs batch-size\"},\"id\":\"1230\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1305\",\"type\":\"Selection\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1228\",\"type\":\"HoverTool\"},{\"id\":\"1249\",\"type\":\"PanTool\"},{\"id\":\"1250\",\"type\":\"CrosshairTool\"},{\"id\":\"1251\",\"type\":\"WheelZoomTool\"},{\"id\":\"1252\",\"type\":\"ZoomInTool\"},{\"id\":\"1253\",\"type\":\"ZoomOutTool\"},{\"id\":\"1254\",\"type\":\"UndoTool\"},{\"id\":\"1255\",\"type\":\"ResetTool\"}]},\"id\":\"1256\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"FinalObjectiveValue\",\"@FinalObjectiveValue\"],[\"TrainingJobName\",\"@TrainingJobName\"],[\"lr\",\"@{lr}\"],[\"batch-size\",\"@{batch-size}\"]]},\"id\":\"1228\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1249\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1302\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\\"32\\\"\",\"\\\"64\\\"\",\"\\\"128\\\"\",\"\\\"256\\\"\",\"\\\"512\\\"\"]},\"id\":\"1232\",\"type\":\"FactorRange\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1184\",\"type\":\"HoverTool\"},{\"id\":\"1206\",\"type\":\"PanTool\"},{\"id\":\"1207\",\"type\":\"CrosshairTool\"},{\"id\":\"1208\",\"type\":\"WheelZoomTool\"},{\"id\":\"1209\",\"type\":\"ZoomInTool\"},{\"id\":\"1210\",\"type\":\"ZoomOutTool\"},{\"id\":\"1211\",\"type\":\"UndoTool\"},{\"id\":\"1212\",\"type\":\"ResetTool\"}]},\"id\":\"1213\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1250\",\"type\":\"CrosshairTool\"},{\"attributes\":{},\"id\":\"1206\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1194\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1251\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1207\",\"type\":\"CrosshairTool\"},{\"attributes\":{},\"id\":\"1192\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1252\",\"type\":\"ZoomInTool\"},{\"attributes\":{},\"id\":\"1236\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"text\":\"Objective vs lr\"},\"id\":\"1186\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1208\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1253\",\"type\":\"ZoomOutTool\"},{\"attributes\":{},\"id\":\"1238\",\"type\":\"LinearScale\"},{\"attributes\":{\"callback\":null},\"id\":\"1188\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1254\",\"type\":\"UndoTool\"},{\"attributes\":{\"axis_label\":\"average test loss\",\"formatter\":{\"id\":\"1295\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1202\",\"type\":\"BasicTicker\"}},\"id\":\"1201\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null},\"id\":\"1190\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1255\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1211\",\"type\":\"UndoTool\"},{\"attributes\":{\"callback\":null},\"id\":\"1234\",\"type\":\"DataRange1d\"},{\"attributes\":{\"data_source\":{\"id\":\"1265\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1267\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1268\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1270\",\"type\":\"CDSView\"}},\"id\":\"1269\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"batch-size\"},\"y\":{\"field\":\"FinalObjectiveValue\"}},\"id\":\"1267\",\"type\":\"Circle\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"batch-size\"},\"y\":{\"field\":\"FinalObjectiveValue\"}},\"id\":\"1268\",\"type\":\"Circle\"},{\"attributes\":{\"axis_label\":\"batch-size\",\"formatter\":{\"id\":\"1301\",\"type\":\"CategoricalTickFormatter\"},\"ticker\":{\"id\":\"1241\",\"type\":\"CategoricalTicker\"}},\"id\":\"1240\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"source\":{\"id\":\"1222\",\"type\":\"ColumnDataSource\"}},\"id\":\"1227\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1295\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"data_source\":{\"id\":\"1222\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1224\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1225\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1227\",\"type\":\"CDSView\"}},\"id\":\"1226\",\"type\":\"GlyphRenderer\"}],\"root_ids\":[\"1271\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
       "  var render_items = [{\"docid\":\"c55983b1-780a-4a94-ad11-457acaea6dc7\",\"roots\":{\"1271\":\"769900e0-363c-4f71-aa95-68870f909480\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        clearInterval(timer);\n",
       "        embed_document(root);\n",
       "      } else {\n",
       "        attempts++;\n",
       "        if (attempts > 100) {\n",
       "          clearInterval(timer);\n",
       "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        }\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1271"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ranges = tuner_info.tuning_ranges\n",
    "figures = []\n",
    "for hp_name, hp_range in ranges.items():\n",
    "    categorical_args = {}\n",
    "    if hp_range.get('Values'):\n",
    "        # This is marked as categorical.  Check if all options are actually numbers.\n",
    "        def is_num(x):\n",
    "            try:\n",
    "                float(x)\n",
    "                return 1\n",
    "            except:\n",
    "                return 0           \n",
    "        vals = hp_range['Values']\n",
    "        if sum([is_num(x) for x in vals]) == len(vals):\n",
    "            # Bokeh has issues plotting a \"categorical\" range that's actually numeric, so plot as numeric\n",
    "            print(\"Hyperparameter %s is tuned as categorical, but all values are numeric\" % hp_name)\n",
    "        else:\n",
    "            # Set up extra options for plotting categoricals.  A bit tricky when they're actually numbers.\n",
    "            categorical_args['x_range'] = vals\n",
    "\n",
    "    # Now plot it\n",
    "    p = figure(plot_width=500, plot_height=500, \n",
    "               title=\"Objective vs %s\" % hp_name,\n",
    "               tools=hover.tools(),\n",
    "               x_axis_label=hp_name, y_axis_label=objective_name,\n",
    "               **categorical_args)\n",
    "    p.circle(source=df, x=hp_name, y='FinalObjectiveValue')\n",
    "    figures.append(p)\n",
    "show(bokeh.layouts.Column(*figures))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C. Deploy tunning job\n",
    "* Deploy from best tunning job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()\n",
    "\n",
    "def get_data(data_root='./data'):\n",
    "    transform = transforms.Compose([transforms.ToTensor(),\n",
    "                                    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "    testset = torchvision.datasets.CIFAR10(root=data_root, train=False,\n",
    "                                           download=True, transform=transform)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=4,\n",
    "                                             shuffle=False, num_workers=2)\n",
    "\n",
    "    return testloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2020-12-31 09:27:00 Starting - Preparing the instances for training\n",
      "2020-12-31 09:27:00 Downloading - Downloading input data\n",
      "2020-12-31 09:27:00 Training - Training image download completed. Training in progress.\n",
      "2020-12-31 09:27:00 Uploading - Uploading generated training model\n",
      "2020-12-31 09:27:00 Completed - Training job completed\n",
      "---------------!"
     ]
    }
   ],
   "source": [
    "# deploy best tunning job\n",
    "predictor = tuner.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAB5CAYAAAAgYXpDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO19aZAl2VXedzPz7a9e7V1d1XtPd88uzYxGIwmEEBLYIwESYQssTMCELcdEOFAYHEQYYX5gRfgHhB1gHIHlmEBCAhMIWRJIFjJGjHZgpOlZpZmeXqbX6q6u6tqr3v4yr3+cc/OcV0t39UJXP7hfREdl38yXee/Nm5nnnO8sxloLDw8PD4/eQ7DdHfDw8PDwuDH4F7iHh4dHj8K/wD08PDx6FP4F7uHh4dGj8C9wDw8Pjx6Ff4F7eHh49Chu6gVujHncGHPcGHPKGPORW9UpDw8PD49rw9yoH7gxJgRwAsCPAZgE8CyAn7XWvnrruufh4eHhsRmim/jtYwBOWWtPA4Ax5tMA3g9g0xd4sVi0AwMDN3FJDw8Pj398mJqamrXWjq5tv5kX+C4AF9T/JwG85Wo/GBgYwJNPPnkTl/Tw8PD4x4ePfvSj5zZqvxkbuNmgbZ09xhjzpDHmqDHmaK1Wu4nLeXh4eHho3MwLfBLAHvX/3QAurT3IWvuUtfZRa+2jxWLxJi7n4eHh4aFxMy/wZwEcNsYcMMZkAXwQwBdvTbc8PDw8PK6FG7aBW2s7xpgPA/h/AEIAn7DWvnK959m39AUAgLFJ2pbNULdMIN+XVqsJAOjEbTomm033xQn91iZiwTFBDAAIQtXndon2gfZlso10Xwh3TTlHnHQAAO2O9C1J2HJkIu6PWJKavE/blhIelzHS2mrRGOI4Wjf2gPvWSqStSt1ArRWnbaX7noDGhz/84XS70+msu+atwHWfz675q5sC3UatgWvUhjjj5i9Rx7t5lpNczZtqo3674z/2sY+t27fvh3lu407aNnflMgCg2ZA1c/CuQwCAgf4KACATSn+yGVp4Wd3G6zkyao116gCAcinD55C+RrwdqkW8sDAPAOjr60vbMpkMn5eOM4Gco5O0AADBBqJaYKSxViXzZhTRmszn8+m+VovO0eFnEAAK+QJfS/r2u7/9W13n371nR7pdHjlCvwvlua30lQEAK01Z19XlOe4v3e9ELYaIB1GIcmlbPuRXmHpu0weQm+JEzu/aEtXmruHGTtfnudxg7Ri+fybQ74V4g+Pot7kc9TcbSL9hadtkZf5qc8cAAF975vvrzrUZbobEhLX2ywC+fDPn8PDw8PC4MdzUC/xWoMVSlLV1aWTpM4dS2hSAvlRRxJK1lij4q2oy0th0UkMiX7iIJbyQmyJ1DpOQVIyOSBlOGk7UOVqGJJM4pC9oS++LAz6XfI0NS/F51beIJZ8goo7H7bbqSIeHJOdwEmcYbm7xCsNw0323Cjcq0ev5SOUkJSUmTmSyPAYr+5xGZCDSjpzl5iXwjVAu0r0NrDwezSq1JS0h4vNZOm+pQMdF6jJu7eTUIitk+b6rsTRjdxytq6xaJ26KokjurZPsAyXFu7nJsVaql0m11uZrCpz2aiHnDfhiGZZCnVQPAO1mk8enxsJSJa6yJhIrUnwnHKRzZeSZjkOSwIOMksDrq9S3uMr9kPM1LR3XVpJvg+dXCeVotUlLCviZqNfk3eKeEz0+pxEHgTyH1mkuPJla4+90Yj5GrmmMez/JmhkcpDHnCn18frlniVvXOelHvFrG9cKH0nt4eHj0KPwL3MPDw6NHse0mFMsmBlgxXVgmj0wsKl7SJpUmLLCZQqmhznqgiYQsq0gdKypK0g67jnOqEAAYu4ZIA2CYcLGhqIL1mHS1y3OkblVbohatrlJbaOW8fXkmsxQJVykSAVTI0TiToJXuC1JziYzdjaCdbK72a5PA31eZvK2ct8tc4Y7v0jXdLm3yoTlvtmk+Iq03x/Tb0Gx07WSDtq3hamOJ2IwVKDNWNqRrZQJpywVsHnP7FAHZrJOpJQwV4RbRfW83hQgNwCazDrVZI49kzKaibKYgx7t5UGvMkbkxmwF1vMXclSsAgLGRQTmezSVhVq4V8rXcPCtLDiI+vqlIXUewttvSthaBlX0x9zdWz0FsaMz5PunH8L4x+u3SAgCgXFtN97Ua9I6Iy/I8Jv0U2d2Xlbl31w3YztpqyvPlHB7yebkv6ZSqNeHWsfsbKJtth8ec6OXHl89GsnYLBSZ64cyAYqJJnHlWy9A3YKL0EriHh4dHj2LbJfAoZsk7lK9fwJJELlRfd8cQ8Zcw0EwN/7SjJVRHymRFetm5/24AwPLiLABgdk4klUxE0nYA+TK3OjQ9dSsBSMfOkURjc8MAgHYopEyLJYPVpfm07eI0SxJ5JVlNLQIA9u6kaw73aSnNuRbK2J1wEdv1rkoOWvK9Fe6Dt0SKT/uttAN2tewo8aXNmtDJ06cBAGM7xf0sYTJ6dEgkyDwTP8lN9PFqc5RlKTvpiOQWsvSUUQRahtuCmNZRNqOkupBdVZV2lQno3iZGaVwJu8c2mMxU66nBYy8WZQ2HjtnU4h/PQ5VdHJ977vl0V5s1gcHKm9O2XI7JfDUFqSsra6eBct8z1pH5siZt4oi8zSXwDsTVMQCt9SRUBC5rYaHSxkrMRlaKfI+ffzbd15olaXz8gbulb1fomWsambcyD2ylTkRoXo0lxxp5MCyEYcAkpn6lNIt03qjNmklbJmulRPclt7SUtkV77gMA1Ab607aEtaqY71k+ESI01fhjaQvj65envQTu4eHh0aPwL3APDw+PHsW2m1Ccnm0iSTPr1NuOjlBkwqjFam1WkUNx7NQ5ZWLgc2i/2rf86I8BAJ77278DAFxiUwoAVDsuslJUq3OTMwCAM5MX07bc4DgAYPfYAbpmTtTEFqt/mbJkfew0SO2bm5E0McVBMr9MrlJ0X0Opw2N9pOIVM6JWxm1Sg3Ww2Vr6biMS83ZEYl7d1MJkWUZFzbKPd31VSOvFJVJ1p2fJ9FToE3V4mCMOddSgI+10dOYGnV3Ti60jy+Y6q86RcZMfS79DOLKd2jLKr7rt1OdEzhFWaB6MVX7/7G+cuGjfWNb16jKZ2spFIe0Cnm8dFRlx5PIik5fzy2IaLLCfdEtZOlptulaU1WuG2mKOdO4o85GLgs4qH2fLazaJNzfr6Zl3JsFAjT3u8FiV7cKwiaNh6L5nElkLZoRMa7UV6Vv7zAnqrxEzU8LTVXX+5er5yrY5fuOCItF5PrRjRIPNoWGD50ouieZO6mP9sphK+ww986Z/RMbH120HjhhWsQ8836EixaPg+k2CXgL38PDw6FFsuwTeDOhLu1RTEVosvQyWRWyoMCkUsQSiCabUDUgRKo7krNUW0ravfonyrkwvkkQxvSrfr3MX6bhzlyTFeZgnaTwOK2lbqUJf2kyR9kV5+fLnWErMBzKW2RZFgY3v3pu2NZhcOX2aJPD5RZWTZRedd/+oaAIZdqUzyo1L5C8er/q62+T6ZM408HEDAUBL3cEGEnjMUlbC0oaOFnURblfmltO25SqNta7zX9RoNEGOyOJqXe5tucgSp+qbk+e3qmBcryaSM87lTebbkZcbugAmHPmnXAAj1hgjxRSGhubDxvru8fiYuI+Vq9nqCs3beX3NyEUui7S4p0Lz5lwGX3r55XTfG+6/HwCQaBfHmOY3r11sWROo11jDjeT8HdYAw0jI/Dbn22k2N08RHSvpPOE1bLXMyE4HLe1uyNftX+G5Gh1L9xV27KP+WCEPwa6QdmRn2lTPcG6Ty5RXBcolt8rPqx0bTtsyCfWpoTT4EmuBrRUaX1PnqClwxGtV7ks0TNqBySg3Sc530sc/DZWE3zE09yZQLrO4/mhqL4F7eHh49Cj8C9zDw8OjR7HtJpQrdVIb5ttCYn7jb74OALjviJgifuR+IgcG2V9ckycuaU2g1JGYyRLFfeHMOfIznq+TamOLQ+m+sMxk2ZCo+wWu39lSKURbTJxVBqlvlbL0ceYymUSWFxS5wSpeviCmlvMLRJ5mKqQezkxJtaTy5RUAwM6KHF9wqWsTRX6tQbWmk4GxCqlUR5dqN1SJkdy2S4+pckghSNZ/212UqLZdrLJ678jMgiK6GhyxNqVMKDMLtJ0ogqvN9pHaChG+M7Myf5MXpwAA9x0+mLbdtX839V/5xadkqouk1VYT120dJnAVajNkE17SFvNAwCa7+pKMBWw+sJwEKSzI2LN8r7Jqvk2bTGexNjtwtLFJiVMxH1WrZCqYnpbjS5UyX1Ml8uI5b63ScXnlj35lkYjQ578vZpVSjq556KDMacSmnGaN1l8hUomXmrS2YpVWOXaPWkPNx1qoKXYpXZOuWA3ep57lDJuvcqdO0umf+1a6r/NmNj2ptKyWYzSyK/JsNEDzUOZ4izAnxyclOr+xiljnZHJ9w/IOylxk88sqrcnMmDgr4ALtiypi5mxcofkNi9KWHCHf8AYnwgoU6Z7t0OREyjZor8LJbwYvgXt4eHj0KK4pgRtjPgHgJwDMWGsf4LYhAH8KYD+AswB+xlq7sNk5rtqBfpICanPyLWlniSicr6lk5y1y66lk2e1KER9O4gxDIVkaLZJgryi+aHaFvr7FASIwBkeFWKwmJEmMQEW9MeHRyohU1KiShNJYpeP3KTKkxtL2TEukYcPS0NK8krpYGqnz1z3MSr+nl2kap5ZE6t83whrGVb7Qi3UZaLlIWkGg8jK44hRdgrUjV1yQa1ca1w2+7Ru4J16eIhfLoSHSZgp5kWyaDRpzMSdtO0dJk7JKPKvWaKwlllRaDZX+kwe92pTxddI8FcqtLXVndPvWDbNLIrya92PeJexXBzkJPKek/jKTxf1MPgXsDgkAOb7HeS1wspYUNGQtpEn+uTBIa1nWWl+J9g0OiaZ4ZpK0vNMXLqdtJ049DQBYmCWJc7Uh56i1qcZKBOUWyJL9g3cfSdve9+OPAwB28Xpu5mWcjWqVfyfXrHCBdFNfwWbIhLL+XDpoR2YCklI1UnJkeYGu1Zkkt9uK0iZWLtH1W3mJdrSg94K5PJO2lSaYgKywZgl5lgrsvppdlH43mDjuzE6lbVmew84yzVVuXhwZ2nXWlgqiwSyeIeeHbEEk8L5xIl1dKiWrXAabjrxWa7iVXL8IvhUJ/JMAHl/T9hEAT1trDwN4mv/v4eHh4XEbcU0J3Fr7TWPM/jXN7wfwTt7+FICvA/jVG+nA3W94DAAw+czxtK3cT1/3x972lrStGJKduMUSsJYuDWdri63ky+jbQfWWX3z5pJx3gKS/XfvItcoqW1qGpeykOZe2tVrJumuF/MV85aWXAAAVlZC9WKIvf0nZwS5dngbQnaclZKliiN2/FhfEfrcwT9tnpsRVamKMXKSirIomWIOoIppAzNJzW9eTY9ti+hdil3TBIVritBv4FDoBXXkspgElLl8GlCvnALtitdvqXCyVFctiU3QSuOHgLKNctnIF526lyoQxsdFlM1zXN7lmpvsQ3r25CH7h7Fnut8z3yjKtu7gtmsDFi6R9LPAaqK6KPXjHMEnN5ZIE4YRcjKSlMvhFnKsn4Fw8VSWdN9xgVGGJ85eIPzkzKTxBtUW/zfezK1tJJsatxFJWZLWpcxT8cunSdNr2rW/9DQDgXuYaRgdE4qyvkmTvyp0BQPteykeyurS54p3Lytitk8YTpRKzBhMot9dVDrxbffSNAIBK9KZ0X22F7kFb5U0yOZ4bVW4wU6DrVtldUru/tjnfSEY9G3WeG+3EV2e7fG2VrlkqyFgafHyuLM/5UB+9e2L1rljltQt2ayy0VUZD7pP2+G3fQG6fG7WBj1lrpwCA/+64xvEeHh4eHrcYf+8kpjHmSWPMUWPMUZ2n2MPDw8Pj5nCjboTTxphxa+2UMWYcwMxmB1prnwLwFABMTEys0xGK/aT67zsohEqdLQp7DxxK20ZYDV88cxYA0NbRWx0yRTz2jp9K2/YefBQAcODBs2nbcy+Q2WOwTCaJSzOSCyVit6KcLibAvV2tCjm1OE9q5FA5ow+hfrCZZGRUcqG4IgWzC2ISMRyt2McuiFGoiAxWoV+/MJm2jQ6Smn14t3JlWoNP/OH/kvNzPzJKnSv3kQp46IAQt29+A7k5ubKNVpl5HClotb3E5ahRZhJHsGVzdH5NTmazZBIZHlTujK62qaoxmObYyNA5Gh05/yKTuosqdefKEqn0be06ycTjMLuCHT4kBFPGRevpwuVBl0GlC9/622d4uKqgiCOe67IWzl4moi2tXanEoUGuVF9SpG6Oj8so18KIXdwCrolZUwRkxOewKu/P5XkivtuKjS72Ofc3zhe0qtwf+X40GtLvSh+d961vejBtq3IK5Aa7zJ4/L6aR119/ncauXN7OzdHc12ty3ignZDwAlEriENDheWjH+p5xYRVF3hk2KRXGiKhcrspYrizR2I1yj21xzc+sJgMX6Tcul1IuK8/BMq/xfEa9+lyaXxWJ2eToYHDN26W6rEmXhqaoolX7dpPJNtRmvbSeK98rXbvBvTnUokxuwI/wRiXwLwJ4grefAPCFGzyPh4eHh8cNYituhH8CIixHjDGTAH4DwG8C+Iwx5kMAzgP46RvtQJgjIuDS9LG07aE3UfL5Ur980cMVIoxilgIiVQ7q9AUiGt4+eEBOXKRgj76SqiIe0bUK7LaXz6pS1vz13TUxnja9ypJHVpExy0ykHNhDGsORe+5L983Pc/GGigQEXGL3JqNIk4FBklqXWLrU+UMKRfptfUX6ffI8B1coImpMUj/Q8TUVbFSn7YwKqllhAbao2uJ77wEANCyTPUoCz7EkpKVWV5hBZ+nrHyJtIyWKlPuhc4sKlbTtIqu0rJGwNHKWA60uzohCNz9HGk+9LpJb3GRJU+VMcTk5du+h4Ki9e3an+0rpWtEk7eYS+IsnqR/Fgmg8ljW+ZkfuSz9nlXRkXUtJuVdW6R6Eaq768qRxdWIhrQ2TdiH7mplIAsNyVZIcW20hR+fnHXmpy3/R3xbnWFmpyly12L10z6i4Ig4P0uJxgUIAML9AeVSGB6gfj77x/nTfJLuKLtVlDb82SfclUOv6gKQtAQBEKhNooY+euVVVIi1ilSVWWfgiDnYJeE0myv3RcIGXSF3TbbVbKgMja9ERS9Za43HkZay0PFeqraNWZabAJGO8Pqupy52S6ShNgBl+ndEwH7sMlnwtteRcIFu3V+/1Zw/dihfKz26y693XfTUPDw8Pj1sGH4np4eHh0aPY9lwomTwRKo2GVoe5/qCKUCyWHClEqr2ul1mOSAX65FMfT9t+8l98mM6hoseyXAvQFYc4cHBXum9mngipxqqowTt3kN+4TpDf5DqFBw8RwXrXISFfl16gWoTVFVETHQnTURFodTZxDHD9vNhKVFj/IKl/HZWBPwxofJOXxLQw9gZ04Wf+2T+XPjK5V1L5VxxpUlCmJ5eaYXmZ85N0RLXPMKkWKf9Xy6poXflH24TO56p2a+I04uMzGR3hud4M4/xfG5w/pKRyTAxyPpq4JX3LhzSuxTkxAUxePAsAOMTEdxgoU5F1FddVyt2ruNwus5nOaqKQffsLoczH7j13Uf9d2tzLstZm2fQzNiYetrkRMutUF8WfOuFI0/5Bsj/kchLL0OAh1zpiQsnzcxC3ZY2FTAa6IieZrCoskaftxx4Rk8iRfRN0/pas9TOv07heP/4qAOBtbxaCc88eOv78y5Kzpx27nESb18TMqn5kuSZsYsVsWWDSuqPS9q5wJGrMRGW+X0w/YyU2aSmyTyq+q7S9cDU/6a8uRLERLD+b2oQSs6+5S9sbqGtmneFGJVpq8jtF516K2IQYcwX6rrq1/NzouqTalLpVeAncw8PDo0ex7RK44QitmpJ8GyxBZnQehDl28eF8JxkspvvGB+iLePKYRF1emjxFGzUpZXZu8iwA4OGdFP25a58wgRMzJAFVT4mUMZQj6a9vQMokvf76GbrmBEnvi8siHbX5Sz59RUlYjtxQroI1lsAN50bQ1EXJZTdMJLIya2g+WrOXsRmStkgIqQSi9pezdN5CXua0zpnkam3qx9nTZ+WaTGLuPbAvbTtzgebyS3/5dNrW5gyQec53UlTnd9Fr/RWJ6hvoJynq4YdFhRgdIanzrt00p4Fy33NSlCOaACGn6jtEOpsYp3s1sYtIaJ3hrsauZl0ayVVElwwT66M7JtK2PBPIs7Pi3lnlqGAXTtdQEZb9o7S2dilX2L5+GmdlRKTyOSa+Y5bI2qpCmXNZrCnir9V2BKVoJFmX8TJH9zhjRUPawXM/Oij3IM+E3OigsI4VdrWbO38eAHDu9bPpvp1DtP6Xpp9J2zJMXrfCzV8hkcr9EXKWxbzKj7I4Q4Ts/KrkILkyRfM72Efr/4H7RBPIsPbdVARumzUATcC79e+KnASKWHdSsC4FGKfEqWYZu3Pr6EynSM8hz1zEx+u1636TcZqRftD59IFyiYyv4tq6GbwE7uHh4dGj8C9wDw8Pjx7FtptQ0lSwSh0ZHyH1SavjX32ZfLIHOan84SFRafI5JnEi8YW+MnOWTt+UiLK9d5GfeMjnLVaEMBoZI4Jpbl7U1SUmL3Xh7R07SP2N2LzTUGSjS1JUV+p+h3/cUSdpNDlVZYe+n8NKpTZcKy9rZCw5Jnli2x3ppvHn/+ev0u2EE9QHyoe2zIRwnzJn7D9MYx4dJpPB8LhEaQ5xn/IqGdPiMTIvfe+Y1A2tW1c8gv4fKfW2wr89tFfMMG977BG6Vkl8rEushjsNtqXmtMO+zbUlMZm12Y+6oKq1DwyQ+WCak4fNqqIQBY4IHNsp81wsqhiANRhkk1mozANNLlxhlMwzP0d9Wl7mtMDK5BdyBN+5i5IwqrJM5o/+fokTcP7fTSbxjSL0ci5asCT3vWBd5KbOjUvPRKnA5kVV+X33MM1LURGKVa5231GmGVfs4gCbfI69djrdd+QIJa6CIiwvXSLf8PygmLEAvd1N2rniIokyZ6xwTMWVK2IaXFyg8554+bsAgNde+rt036FDFHOx/9C9advgCJuBlPnBpU52xT20YSJMfchV39LCJqpqPBOQUjhGkaR8vObB08jlDdjxlCTtShbHZ1X3W79LtgovgXt4eHj0KLZdAndRUv1lIZgG+mjbqJwby5YkidkF+hKO9EnXS0zAxIFIHmcvnQUAjA1K8vd9/AV37lnffU6iPy9OkaTeVxapPMNuTq+cOq967CIJ6W9TfTVXOQJuQCXg77BYOTWtEs73UZ8idlUqFkXCcvlD0BYiNK5S38Z2bJ4L5dkXvp9uFzJEKDabQrBmmYR7y1vfnLadu0iS9BxzSA/cL65mWSYga02R4jOsuTzyiBCQDY70y7K0ePigRMPezylHJ0ZE4qwU6d4mym30wmWKApxZ4GIWs1fSfVUmtxcXRQJvcUrXjHKJdLlYXKRuWxGKxQGatwcg4+vv33wunSRdU5GeoXEl6UTqjzk1acQRvokVeSibo/OPjEhkb5nXeF65ZvZzvyO+Z9q90rKrXke5d/azi2WgohcTTpsauejFpkjW/ZyAxXZEK4xZq2mpSMI6348ir81zl2X9vfo6aXfNpkR4ths0vzbUVPnmcFJrPi9jv+duigQ+dK+489ZWSBp/5XlyyX3hqBCn3/omaYDHXpW1fuTehwAAh+8WqXxgkNabI3fDrj66+d0gF7EmR10JuM76MoYuOjNWpGeSujNujq50zcaVgZQ1rFNObxVeAvfw8PDoUfgXuIeHh0ePYttNKC46bucO8cl2NfISRQaO7ybV/CibRhaNpGy1IanZ/SNCFPZX2AczL6ryfjahlDmF7R984o/SfTW+1nJdyK8a++HqzJM7OVKyMU/qXDWnr0lmnteOiz/69DSZA5ZVdObAAJ2wUiJ1OFSkU4aj48LaxbRttET7+/OioKmknACAKxeU//oQmYF27xbS7r43HKbz5+Qcr7xIRNEYq7VlVa1nhusDlipighqu0HHve/wdaVvADtX9/XTcyLD4r89z6t0z52Q+lhbJrLO8JNGnK0wWL3La3vllibDsMCGbUWl+s1wBJ1CRa/0VGtcAR24OKnNTjk1U2YKYqlbrQhKvxTD7cGvf+jJXV0lUOtRMQPOxg/3FjYpCzbLPsjPtAECeoxFDlXfWmUzSKkTKhOJ84GtVWTsuIjCnFqVlc0ptieb74lmZ73l2Ph4oyPFjnHI3n9c1ZNkkEpH5KCoK2X2F61PuGZdnro+rVS03NyfeEpUm1iW9soFuo76Fyjd8YJjSsr79nbR2Dx0Sk9y3v/F1AMCZM/JsVF/g53ZZTGwPvoGq+ezZQ+fS6ZrjDq3xWPUtYVNtVxWqtP6r+yu7XL1YTWg764f2OXeEZnqtLhKT33HKDKNNMluFl8A9PDw8ehTbLoE70q4yKBJ4J6Zu5SJxyzrChQiOPkeS1XJGItwSQ9Lc2C75kr96jNyPfuCH/1Xa9necqL9aJSmw3ZKCDjOXnWucfNNWuYZdpKLeBgOS0HcV6BxLV0Ta6YQk+Y7tECI0ZterupL4GnWSOKtMlnUSkbDaDYpE25ERSW+iTJJSsyNtayXwiydeSbeXmej6yX/yb9O2xx+n5JF//VVxN9zB5N4OrmJfUK5peY5OG+sXSayPt/PKfa/DUouTNHXOl8vHSVI6PyOudC0uzBHlJW1qXx+RvjtYImy31hNHGZWU3+WM0Lkj+vpoLJVKH+9TdRY5H830tNzvRmPz6lBFlj7bimgtsEvkQEW0miRNbUwEZEHV+UxJKiX9JZbbtNzkimm4v4pc6/D97sTS1+U5GoN+cDMsga8ukbY3dUmij8eGaCwDJYkmrrH0nChNoMNndMTpLi5QAAB3c53Mh+6TIhknTtPz8sL3xBFgLXQK5YALLgSRaNUZJvFjFb3o0rEGTOoePiKEecJut1NTn0vbFmZprCeborVNX6T6uncdJpL03vvlHDvGiFSO1Lul0+ZiEyrFbMw1Xt193LAASFdOlvX705TFPA/6FGnxFCXad0V7bhFeAvfw8PDoUWy7BO5yfwyOiITQ4a91I5BCAPkySxKcwe/8BXH+f/ubyT2ssSpfxGIfue1NXZTcFYSO61cAACAASURBVKdOUDXujqtWrbyLqmx37RsWt6+lJZJ8+ssicd59hHIzPPvSawCA54+dkX78yHsBdGdRPH2KJPRFldHQuSA26iR57xsTya3AQRtDQyL52ogkg05rczejhipt9eAbqY/veve70rbhAbJN/+BblP2aJbc+1gQqZZGKQy5S4KqmA2Jr1Un2lxbI7lphiSZRGVgO3v0AAGDHbsnYOL9AmkvfgLgWusx2xq6vGO7sqK7UFwCssk3YqhJYrlDAhSmy3TstBwDaXOxC50cpljYP5KmyttSnCjq4oJ4ZledmmYOLEs5aeMgFvAAY4PwhYUZLl7SttZQW1+eqMffRaEq/Oy2aK6MKQNgmHV9SGsnAAGkwhSzZqCMj62SAtbf+PlmTLT5HTWVbbHEG0IADSwaV5lXkLJ6TimdxheHvv/tw2nZFuX/SubQ9n+3dqm9Z3p3oB5ElU2cjbiltbPee/QCA/fv3p23PTtP97qhyb1dmFrk/JJ0fO/Zyus8FKt11l/R7bIzcGPv6hO8BB9Q1uNp9rJ69DGtcOmjHuRHqOB5rtKsijSo9fVoAQhDeQEGHa0rgxpg9xpivGWOOGWNeMcb8ErcPGWO+Yow5yX8Hr3UuDw8PD49bh62YUDoAfsVaey+AtwL4RWPMfQA+AuBpa+1hAE/z/z08PDw8bhO2UlJtCsAUb68YY44B2AXg/aBamQDwKQBfB/Cr19uBhGsM9g9JEv9qndSWWiwqhyOsXK3DE68o17QaqSrlkuTy4Fz7OHdC1L6LTO687W2UTlan6ezj9LBDE+K2dH6ezCT1pkrmXiJ1tTJKJM/DfVJ78Qqr12fPvShjqZG5YXFJrrWDq9b3W+rPvrK43u2ocBEEIyYRl0K0pFRSccIjHLznoXT7g7/wb2h8sajZx08RkZgYlUOGyc42q3PziyrpS+LywAhd6gp/JxAiamWZehJOk6p7SdWzdIU5koaQQyUmTE+fFNPWGU5h6tzwhkZkPpy6v6Sq0s/NEpFnlUkkYPc0E7i8ICqylwnTvE6lu7qWBhbk2GVxblbG8voCXdNFMQLAwCApnePjlI+jpaL22i0ywyRW+rjMZq66Mu/EHCEZsnlK1150ZpK8qu5eYPfBhlq7CRN/pTK7pap1kuUoRE34OkK4oUg7V+ndkYhtVbRjco4iZGuqhqYjAXeOy/pfi1CZENJtdU0Ynq8u9zr3G7Nun4vi7OsT805KLnYV63AmObrWyoLcxxc4JfMrLz2btg0N033cuVOI253j+/maZFYZVqbVUS5IaxRR7u5zR5n1Okxypm6E2hWRzVdWmdNsstbkcm1cF4lpjNkP4GEA3wEwxi9395LfsclvnjTGHDXGHK3VNmf+PTw8PDyuD1smMY0xZQCfA/DL1tplnbnrarDWPgXgKQCYmJhYx8KtcCKOgsrklmZmS1T5LyY/RoZIOjsRSLa0mXmSbOZC+YL1l+krec8DQkycPkuSnkuar4nFw4eJ1Dh84K607dwUSRyvvPK9tG1uloNCOOn/oHIdm3yFJPapWclBYpiIDVVA0fgecsfax1O4t08krDyXZmo2dKABSUzazWktPvBz/zLdHtxJUtFL3xcp15FBLfWVj5lUc6XDNIniSlXFWkLgtqDrs8+5RzhL5OycuAw6NzgVu4GBygD3RyTZ+TnWNlgKnJ0VwrLJ2kdHuWHGXNYuVLlQinma55xzMdQVw13yG4h0VFBZFtdikYnZSxfFHa/E5PI9qsCAy9hY5PwujbpoTQsL5G7abss4a5yrpKjcMPsrtO5LOfpbUORkxM9YrEjMTqfF51XZLV05r7T4gCoSwFpsWz15UcgkXKJcWznb4twV0jRm58Tl0mUNXFD5aJwmlesTbWktjNUSOP3VxJ5hqVXnCEklaf7rCEMAqK9SPy5flgIQly7R9lJRjsvwOnKkfEnlXylGdJwmtC9yEYmTZ+WdUq9T0ZJOTOcaGZXiHg8+SAGBhw+JxD46Smuh0i/OGLkCaQoWfH317HXSJIeKSP77IDEBwFCO088B+GNr7ee5edoYM877xwHMbPZ7Dw8PD49bj614oRgAHwdwzFr722rXFwE8wdtPAPjCre+eh4eHh8dm2IoJ5QcB/DyA7xljHDv3HwH8JoDPGGM+BOA8gJ++kQ6cPkVqy97Dkg4yH3BazJYQTRGrQUJkCOlZ5iIF99wjfrh//VdfBgDUlsRfvDhMZNOpSVIW9uwW0vPA3VRoIKfU8oN7af/ivBSFeJXrbiZMkEwuCNmzzORrIxZz0PIimWl2KILk3By1De0hc8JcTvkkJ0x6KnOJjbgWYCLq+Fov5hdePJpuv/w9uk0GYppx+SYiXXQgTY2a4WNE9Y44/axO/+nykWRVfwP2Ew8t7atkxZs0YDNTO1TqPkemKrddZDlXSbvG/slVMUG1mOQzbRWdyTacliK5Y462rK7Q8UV1H0f7qR+RMl04S8VGVObQKK2TQVVowxUkiNR8rKwSkbi6Sv3N5cT84UhAnY50YozI61xe1H1HXlrOx1FtSI8aTBAvLkh+nrl58rWuK3PNvZy2N8O+9d0FDLhep1pPTa7lOZlGH4sPd4vNU7WqnH9pkUyJWRVV6sb+9Fe/mra94y0PowuqWEHi/Ls7KgKSTSzKHR0mNe/QvlBFpr70/HMAgNUF8TcfZv/2C1PSVmEf9iw/N4mKYK6U2R9d+ednIy6EkVNxEAGbZRfIbHT2jEQ6Ly7QvD1/VOW+4biJPXskWnWCC6SMT9CzPzEm75sSp602BVWvM9g8NmEzbMUL5dvYPM3tu6/7ih4eHh4etwTbHon54imShvc+8FjaloC+fkaTdvwFX2ZCZXFRSJbhIXKhe+/jP5K2PfRGyoPwmc//WdpmOK9BP1cH3zUhLlBlJtfCjkgeQztpesYPiBS1xMn4n3+RpNypVeW+lCHCtH9ciJ2RQ9TWVQiA3faOc5GKU5dFQs0y21NXkYdVnoZOIlLDe9b4/HzrG19Jt2ucmS2bUaW4io5ElVseWs5/4ap4Z7QETv3I5xTBym54WZXFLirRWPNZGmdO5XNwqTaMyqLoyOi2KhTRYIIylVp1BBsfr0u1pSG0SuIdKNF2f4nGVC6IlJvL0PkyRu6jUe6Aa9FmUk27HUbs4hh3EXOunBzPnxJz8ixl16syzjpnYKwrH1Cn6QQZ51Yma/74sVcBAOfOnk3bXBSxVe6JE+NE2A9xRsi68vZy24sLQkDOMUlbVxquy9njPMUWl0ULCnjui5GsHZdv5fJl0XDXSuBtVUTCkeimI+dwUZ/aec6C2hzpuboqk+WKh9x9RLT1Rx56FADw3MtS5OGZZynL5iIXA4k7cg92jBMZ+fa3vz1ti/g+nz0nLsfPPEO5lB64j6K8K/3iDDHNY56eFsLerd2dY+JueODAfro+OwJUV8QN0zkEZCKR+hsb5AC6FnwuFA8PD48ehX+Be3h4ePQott2EcmKJVPTZWKXizJBKHbSUypG4GnL0d2JcbAg/9ANEQOYzolYe2EeRlT/+gQ+mbZ/9s7+ga12m804tifLWaJwCAGQhKux8nbZPnRM1Eazm2FEy0QyOiTkhrYunoh0TNjckRlR6l7xpiSMl8xmVtItTulaNSsbE5KFNtIrVrW6NjUp02lSdCJ04FrW5wnU6I9W35VkiZ1eWq9wvUTUTp/5uFB2mzCSZAt0Hm6Hru0RkABCwDaWoknu5yulxe715DJw0yWTFFpFnMrKgzBlDfaR27lE++LvHyf/W8ZTNhqjegaX1FKnIuYEKrbua5KZKceIEpUi9//770rYCm0T0dARMDSUcfTetolBdcrRmXZkp2CQYKzPJwUP7AQCjO6j/utBAhs02AyqxlCNAdZlH58P92nFKo7qqCkC4fTqGIGETUXVF5qjG/axxtGhLmbhc8Yjz00IUuhql8VXqONquCEvrNlK4KEoVJIrEEZ98qwqqXuwPvfPdvEt+4Io1HHlITLAPvInqvrqyoYGi8FzBkYMHJd4j4jndf1jSzk7sJWK4wBG9/cqE4sblCpYAYibZMSppsV1yrJBNT4Fia2N2SGgru1tiNp/LzeAlcA8PD48exbZL4McX6RvyhW9LtOND+0ga2ZkVA3+RpYDxnfSFGx8RqeSug0xGWpEapjgvySc+/Rdp23MvEinkIj27AhutI5HkHHGOrhFrYo5d8zpMiHYCRfK52VSlkRotPq/60kZMaIYsbVmVK6TDlE5Gfa1daa1We/NILdsWib2/RBLFiiJC2zFJZffc+4D8ZoKkkRmOvptR0XernBdFpz9wkqON5byliKSMe95IaTovqVJpV5ZJwq+3RCKscyEFHfWZY9fGEmsaAyr3xyhXGB+fEMnm0C5y89uREzF0lV0P59nNLszK/BVLRFqXVcTrMOe/uHRGiCuHNkvvjVXRYAJHHioR0hVriNlV8OTJE+m+lSVHJMsj5opeREp8TjgkL+BIVijXyGHWmjQ5WuMUxPW6zOmFC5Ndx6ngPlh2uay15J456bk6KxpuhvvpSth1VKRild0IO8p1USIZN5ca60r7CNklMrIqQpaf146KkO3wPLjz67JsTqDvKA3GlTdrqRwkE3s5n1HCKVsTVTSBn/Mz58U1s95yeXRUgZD+A13XX1iSa0YsUZcq+2WwLp/Qkoz50vQ8n4M6nlPpsV2AqSnL+mgsbF7mbzN4CdzDw8OjR+Ff4B4eHh49im03oayyWvHXz4v6eeJ1is58z5uERLprglT1M6cpEvIdbxZTQJ5V75WWqGef+UtKF/n8q5KQqOaiwNiEEajUnU7NCVT0mDN7xEo9a7Jpo80qnlG+xU2OaNTkTRStr99Y5MQ7WbgK2ekuxEwC6iRSHSb8sn1SxWZt6pm5S5K4Km6TKlZX6m3tAiXyGlIVwEc5zWqGq8AUVNapeugqjGg703q1uVYns8s7uCrS/fdKsqfz58k8MbcokaxNR44p8itiYrrArNOIIiwHSiW+styDy7M0luOzktTIMBFV2UFmoUJFCM4ik546TW1ZkVJrUeB71lJmCkcud9V5dP7fbH6oVCQ6OM8+9eWSkHAhj6uoojmdyeLka5QIbWleVPsljpiMlc93JssRoWo95VgfN646vYrmnGGirdYU9TzkMQz2y3pqsbmtxk7qHZUsK0nNJTofKs+H2VwG/OY3vyZj6VBVnFIk8xHzumsrM4kj0l0CL/0stdlUpZ9HRxA2mtIWpxWeODWzqn85NEDm2XJZV4RyFeL18EzXX11t3o05UCaRiJNkBWb9cW4IXeENht8fRTk+aLD5TxHU14KXwD08PDx6FNsugQ+PUH6I+QX5/E1x1Njfct1JAIjb+3iLvnSjOyWK0oT0hf3uUYnG+ouvUiRVM5EvPvhLHATrv1sxS4ZWfYade5iWAlwUZYa//EZ/LjmPgyapXC1Fnbsl5OuHliUKqzQBluK1WD6+k6TFvoqSGmvdEvjO8aF0e/L8JI9JJ8+n7TMnjqdNS+ze565eVW6KVZZ2kriL6aXjVSrhVpMktue/TdXu31mScT7A46z3izTsSDsdZdtggm2JoyM1mXruNYp2m61LZGAjQ9cv7JAxD+4kiSpXoTGFKhKzyG54uaKQ4ibcfOk7V9W4I/fARfEmHaWN8dgdiVlQkYoBa4V1lVOkOU/a4HldjIHnwaVUdflmACG7M3kl9fMlWi2Zv5UFkrgbjVX+K8Szu1N5tebbdU5Jq+qXOsLR/dXkoXP36yjtw7LUms1sTqznVSRwO+T7olJE59hJIFGup86NMuBratI44XwxWup3EamJVVG2PGrr6k6qqvdOeA9UXdco5BTOTYkcTQlNHp6uudlmjVhr1W7NmK4q893vmZaKKrV8joZ6feRC0pYmJvZhq/ASuIeHh0ePYtslcCetZlSWvE6DpKcz0yJ1NasUXPGOR6jCeWFAVY/n4gff+I5k5Kuz7batssHl2I3LSRcbVQgKlTSQfkyVbSzHkptxolCgjs+RlFFQ5bycy1FbBa6ssFTmgiCaStLrH2QXynFJDF9m/8S6CrxY++nde0QynS2zS111clYdwVnplHvYPF83y2NuKXu32F3Xu4l1JeBnnHyZ8k9cWBHJZjSg+ejSYFgqWVX29suWpL5TbBOdVDk0akXWYPZKQv2xAySh5AfElTS9DywVlcuiCRTZHh6oNWavYrtd5jw7tRVxI5y5RGuy0ZC+uXJoLg+GvsdOkwtU8FCGA80cLwJIBsiIbebaZbDNdmCdT6XZpLWzotzV3G0rVdg9VUl+tk3z3FxV1e45N8iSkjid5O3sy0bZuxO7PpjL5YYxyeZFRhJ1H1erxIMUQ30P6G+sFrMLOGqxW2yno1zruHCFVdK2ZH2U57DDNvDYaXvqXrsgJi0cW0v9bDZ0bpi463itmduUj4lVmwvi00VRuq8ZtnS/OffMoC70QtsT8BK4h4eHxz94+Be4h4eHR4/imiYUY0wewDdBNQQiAJ+11v6GMeYAgE8DGALwPICft1aFQm4RKSmkibyQVMGWIlmmV0nNef44EUHvrYlKs2LJtHBxQUwMeVahOzU5R4NVRlfDMFJRcm5fl5uYcW5IcpwNulOwZnLiErbKrlctlZLWmVO0GcGZTKocEVoeEHPJIOdSaKkUmK+xi1lGuU+9aY2WVRkUQm90jPKTTCkTSqrOqd802Uzi6iVqV734KhF2XXv4xG1Wwauzki8jyHGKXuXCdomv8aKqbH8q4vkok1pe2iNFIUYnKKfNMBdZAIAcu+a1VE8sq/m5iKuwR5pIdm2KZLyKr9bls+TSqquEO5Xa6IhaTmfrqpNr9TnL5hqdB8bt1wRhh00Gq6tcs7Spc5awC5vRLn20LrKq+MDYrgk+B0VMLi+I22aHCzRYXYGeb1qtpc0qzjzhfN6w7viMGrsrtFCrKbPeGly4IE4FJ6eoHyVV4zJi20/cVW6A5tRFWyaKWM9yrhzd5kwusU4NxPPsSEajcow4clTbqlw+FX1fnLtrErsoTUVOssmxK+eRK1hh10eOul+2VZ6leIjWxa4HxVW6393S60iJshUJvAngXdbaNwJ4CMDjxpi3AvgtAL9jrT0MYAHAh7Z+WQ8PDw+Pm8VWKvJYAM7vKcP/LIB3AXCl0D8F4D8B+Nh198CRAzpRPgebJCpvgstHcmaGvvif+MyX033veicldT9zSaS/qnPOV9+ojMvkxlJAUbkBZblQQ31FpGdHNFhFMmaYUHQSniaunKSXKMKjzi5jus0dN8BS87BKAn9ljgI5FmclA+LiOQpeOnTwADZDIS8SWY4DRjIqH0jMZJb+uHdSyYTHp3deRQroorRY2lnl8b2mpLp+Lrf2WkMS37/C2slcRSTT4T00rvEDJG0PKJfIHLslBiqfRZvXShip0mQs8UZpUIscn0rP2sXrKiRmmLArnXLlTN399HlZGwusk8jkHE12iey0ZT05iVpXRHdwZHcmq0vecRk8TQLzWsznlDtegX4zP0fX1FkGM6xRhrr6OWubHS0triHhugJXXIELpdWsctGQWlXyqaxFYFU5PieNxiK1Omm/KxgoZDdC61z1lCbFkq+Ka0rn3ipXQXcjrPgMpnBStnb17fD124rET/gdZF3JO/U8pHmNVEcM1o/FMlnd4YDBisrns/tBcsaIjNzvxROcD2q3aJvXwlar0odcD3MGwFcAvA5g0UqY3iSAXZv89kljzFFjzNGNvD48PDw8PG4MW3qBW2tja+1DAHYDeAzAvRsdtslvn7LWPmqtfbSocvt6eHh4eNwcrssP3Fq7aIz5OoC3AhgwxkQshe8GcOmqP94Ew1xJu6ES8Fc5Uiwbij+1SzPpfHm/8d2X031nuD7fYlWYjPlVUoMVF4gSq+MdVqNyqrq6U73zBZVnIXA+uqKqO5/VDpsMjPYPZZUqVhXUW+ynWlD5L1xS+aERMp20FIHb5AIG9ZxcM+HoPF2xfC3aKmKyyvks+gbkmo0qqc26YEDM6l6awVSlMjXrtfwUVqXLtUwAVdlH91uqCMe5GrXNqXwP0RhV6B7fPZq2HRil7eF+mpdARXNWWS5oKCIqYlVe16zMc5RlxNXB8wURFnI89zrK8WpINsjD4ZRNq0w5ltnf1ESjzuEi+WJtAuB1pNedW2OOVO2yYiVuPQkJHDNZ3MrIvXUV6p3pJNGEJedOaSjt143Lal9od7wzP6h+RDwW2xLieWGOzGLt1uZrsqP8wGM+rhVoAtflxdFFQLiJn6VA3QOXMjbRpg42cyUq/bIjkJ01Qx/vTGDaapM4/2xlMnNmo9TUov272cwDTbA6M4x6H7Q5rfPQ3VQ8Ytf+Pem+BtfTfP01iV0ptNlSLUHm18Q1JXBjzKgxZoC3CwB+FMAxAF8D8AE+7AkAX9j6ZT08PDw8bhZbkcDHAXzKUEKBAMBnrLVfMsa8CuDTxpj/DOAFAB+/kQ40WKrMqU9JkyWgTChSaIc/hC5BfVAQKe0sk5eBIlk6LB11FAHZ4IxrVY6E1ESNk4pKWZHSCkxsBkpqcARhoUjX1zkprnAmuUS5C0VMYAxWhGTcOURax86dRNYtVkVSWebMfatLEgU4wIn9Z6/oyMoRaLRVlfUwS2MfHJVrtss0l522yvyWuL9McCoJ3A1ZR+Sl0plm6xzRxtn62ioHSbOf+n3XgJAyg0MUPVmuyNIrF+m+5Zggbqh8Iy12O7RKeg6d+6fuB29nWJPSboSuWIEmxOxVWNoGu95F2n3UuaZpV0QeuyvsoNfTWsmaO0Bd1ZGSPPfOjS9WkY1tnodQaV5tzqcRK3fXUpM0Fyd561w1zTpL7xuUPks2iKh1/Yj0fHO/56cl/06bI0L1LVgHPXTOmRJk5ZoZlw007qpAwT/luVKnsy6Dn9IA86xhDFaE+HYl1FwBEj2nIbt85pSG6/KcdEWf8n1xkakryyqPCS/PJJI5WuJUg9GI9GPfESIqBzm6+uJrp9J9s6co42qk+pa/Sl6ZzbAVL5SXATy8QftpkD3cw8PDw2Mb4CMxPTw8PHoU257Myql4OZX0p+iIjLaojs7NM2EvZJ1gJ2F1q9NSpFPsUkpqIoq2kzRlpXy/FubJdDGvrlnhQgD9Ksqxwr7jeZB5xVWXBoCIVbxQ1WpscvIjVxBAH9epca3Bmkr6szjHYxf2Nc8Rf42rRA+GSv0aGCbzTrmk/MCbbFJSJpRO7HzDne+vSszF3/agKz0mmwVUMqaIVeIimyz6+lSEICfNL+eEjC6xb3g2J+pnizdX2W+9rghZR7TmlbqaDZ3PtKjBwRrzhL7vLSapsllFOmU2n0sXXRsoM0XGme60+YP75maoq6h4Gpmnkj3F64lkF4nsCju0WnLf62w6iesqYpJJzJIyMxX6SUXv8DjbDTlHsIGNI/WH14R2WjSeNkoqRqLKtU2Xl8Ws5yxQes2sRdhRc8x1JxMVgWtB/Q2hUujytkStKgLS2K6/AJBwsrpaJInvJJrapYNW883R0o229M2tddPlS552ks+kQj35+pqgrnBq49EjEqsR8Lvq+LPfoWvOiAk05PunC3NsZNK6FrwE7uHh4dGjMPYG3vo3iomJCfvkk0/etut5eHh4/EPARz/60eestY+ubfcSuIeHh0ePwr/APTw8PHoU/gXu4eHh0aPwL3APDw+PHsVtJTGNMVcAVAHMXuvYOxwj6O0x9Hr/gd4fQ6/3H+j9MfRS//dZa0fXNt7WFzgAGGOObsSm9hJ6fQy93n+g98fQ6/0Hen8Mvd5/wJtQPDw8PHoW/gXu4eHh0aPYjhf4U9twzVuNXh9Dr/cf6P0x9Hr/gd4fQ6/3//bbwD08PDw8bg28CcXDw8OjR3FbX+DGmMeNMceNMaeMMR+5nde+ERhj9hhjvmaMOWaMecUY80vcPmSM+Yox5iT/Hdzuvl4NXJT6BWPMl/j/B4wx3+H+/6kxJnutc2wnjDEDxpjPGmNe43vxth68B/+e19D3jTF/YozJ38n3wRjzCWPMjDHm+6ptwzk3hP/Oz/XLxphHtq/ngk3G8F94Hb1sjPkzV22M9/0aj+G4Meafbk+vrw+37QXOFX1+D8B7ANwH4GeNMffdruvfIDoAfsVaey+oDugvcp8/AuBpa+1hAE/z/+9k/BKoDJ7DbwH4He7/AoAPbUuvto7fBfCX1tp7ALwRNJaeuQfGmF0A/h2AR621D4Bq1XwQd/Z9+CSAx9e0bTbn7wFwmP89CeBjt6mP18InsX4MXwHwgLX2DQBOAPg1AODn+oMA7uff/A/TlV/2zsTtlMAfA3DKWnvaWtsC8GkA77+N179uWGunrLXP8/YK6MWxC9TvT/FhnwLwU9vTw2vDGLMbwI8D+H3+vwHwLgCf5UPu9P5XALwDXLLPWtuy1i6ih+4BIwJQMMZEAIoApnAH3wdr7TcBzK9p3mzO3w/gDy3hGVDB8/Hb09PNsdEYrLV/ZSVJ+zOQEsLvB/Bpa23TWnsGwCn0QMWx2/kC3wXggvr/JLf1BIwx+0Gl5b4DYMxaOwXQSx7Aju3r2TXx3wD8BwAuq/0wgEW1iO/0+3AQwBUAf8BmoN83xpTQQ/fAWnsRwH8FcB704l4C8Bx66z4Am895rz7b/xrA/+XtnhzD7XyBb1SxsydcYIwxZQCfA/DL1trlax1/p8AY8xMAZqy1z+nmDQ69k+9DBOARAB+z1j4MSsVwx5pLNgLbit8P4ACACQAlkNlhLe7k+3A19NqagjHm10Em0j92TRscdkePAbi9L/BJAHvU/3cDuHQbr39DMMZkQC/vP7bWfp6bp52KyH9nNvv9NuMHAbzPGHMWZLJ6F0giH2BVHrjz78MkgElr7Xf4/58FvdB75R4AwI8COGOtvWKtbQP4PIAfQG/dB2DzOe+pZ9sY8wSAnwDwc1b8qHtqDA638wX+LIDDzLxnQYTBF2/j9a8bbC/+OIBj1trfVru+COAJ3n4CwBdud9+2AmvtUPp/igAAAUpJREFUr1lrd1tr94Pm+6vW2p8D8DUAH+DD7tj+A4C19jKAC8aYu7np3QBeRY/cA8Z5AG81xhR5Tbkx9Mx9YGw2518E8AvsjfJWAEvO1HKnwRjzOIBfBfA+a21N7foigA8aY3LGmAMgQva729HH64K19rb9A/BeEPP7OoBfv53XvsH+vh2kRr0M4EX+916QHflpACf579B293ULY3kngC/x9kHQ4jwF4H8DyG13/67R94cAHOX78OcABnvtHgD4KIDXAHwfwB8ByN3J9wHAn4Ds9W2QdPqhzeYcZH74PX6uvwfytrlTx3AKZOt2z/P/VMf/Oo/hOID3bHf/t/LPR2J6eHh49Ch8JKaHh4dHj8K/wD08PDx6FP4F7uHh4dGj8C9wDw8Pjx6Ff4F7eHh49Cj8C9zDw8OjR+Ff4B4eHh49Cv8C9/Dw8OhR/H86g/sGL68EWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GroundTruth:   cat ship ship plane\n"
     ]
    },
    {
     "ename": "ModelError",
     "evalue": "An error occurred (ModelError) when calling the InvokeEndpoint operation: Received server error (0) from model with message \"Your invocation timed out while waiting for a response from container model. Review the latency metrics for each container in Amazon CloudWatch, resolve the issue, and try again.\". See https://ap-northeast-1.console.aws.amazon.com/cloudwatch/home?region=ap-northeast-1#logEventViewer:group=/aws/sagemaker/Endpoints/pytorch-training-201231-0908-005-ea3ace00 in account 843456404290 for more information.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModelError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-2fb019ce10c2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'GroundTruth: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%4s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredictor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Predicted: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%4s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpredicted\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/sagemaker/predictor.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, data, initial_args, target_model, target_variant)\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \u001b[0mrequest_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_request_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minitial_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_variant\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msagemaker_runtime_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minvoke_endpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mrequest_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m                     \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m    356\u001b[0m             \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m         \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m    674\u001b[0m             \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m             \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    677\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    678\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModelError\u001b[0m: An error occurred (ModelError) when calling the InvokeEndpoint operation: Received server error (0) from model with message \"Your invocation timed out while waiting for a response from container model. Review the latency metrics for each container in Amazon CloudWatch, resolve the issue, and try again.\". See https://ap-northeast-1.console.aws.amazon.com/cloudwatch/home?region=ap-northeast-1#logEventViewer:group=/aws/sagemaker/Endpoints/pytorch-training-201231-0908-005-ea3ace00 in account 843456404290 for more information."
     ]
    }
   ],
   "source": [
    "# Inference at end point\n",
    "# fail: findout someday\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# get some test images\n",
    "test_loader = get_data()\n",
    "dataiter = iter(test_loader)\n",
    "images, labels = dataiter.next()\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer',\n",
    "           'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "# print images\n",
    "imshow(torchvision.utils.make_grid(images))\n",
    "print('GroundTruth: ', ' '.join('%4s' % classes[labels[j]] for j in range(4)))\n",
    "\n",
    "outputs = predictor.predict(images.numpy())\n",
    "_, predicted = torch.max(torch.from_numpy(np.array(outputs)), 1)\n",
    "print('Predicted: ', ' '.join('%4s' % classes[predicted[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Deploy from specify job or best job by job ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get job pytorch-training-201230-1148-009-b457940d\n",
      "\n",
      "2020-12-30 12:19:22 Starting - Preparing the instances for training\n",
      "2020-12-30 12:19:22 Downloading - Downloading input data\n",
      "2020-12-30 12:19:22 Training - Training image download completed. Training in progress.\n",
      "2020-12-30 12:19:22 Uploading - Uploading generated training model\n",
      "2020-12-30 12:19:22 Completed - Training job completed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<sagemaker.estimator.Estimator at 0x7fbe9ff241d0>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sagemaker.estimator import Estimator\n",
    "\n",
    "all_jobs_info = sage_client.list_training_jobs_for_hyper_parameter_tuning_job(HyperParameterTuningJobName=tuning_job_name)\n",
    "# track first tunning\n",
    "sub_jobs_info = all_jobs_info['TrainingJobSummaries'][0]\n",
    "print(\"get job {0}\".format(sub_jobs_info['TrainingJobName']))\n",
    "attached_estimator = Estimator.attach(sub_jobs_info['TrainingJobName'])\n",
    "attached_estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fail TODO: find out some day\n",
    "tic = time.time()\n",
    "predictor = attached_estimator.deploy(instance_type='ml.t2.medium', # ml.m4.xlarge\n",
    "                                      initial_instance_count=1,\n",
    "                                      entry_point='inference.py'\n",
    "                                     )\n",
    "# predictor = tuner.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge')\n",
    "toc = time.time()\n",
    "print(f'start predictor time: {toc - tic} s')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p36",
   "language": "python",
   "name": "conda_pytorch_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}