transformer-experiments-lmae2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:38.619584Z",
     "start_time": "2019-08-19T17:09:37.539914Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#%load_ext autoreload\n",
    "#%autoreload 2\n",
    "import os\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.parallel\n",
    "import torch.utils.data\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:38.764093Z",
     "start_time": "2019-08-19T17:09:38.715052Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.nn import Parameter\n",
    "\n",
    "def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict,\n",
    "                          missing_keys, unexpected_keys, error_msgs):\n",
    "    r\"\"\"Copies parameters and buffers from :attr:`state_dict` into only\n",
    "    this module, but not its descendants. This is called on every submodule\n",
    "    in :meth:`~torch.nn.Module.load_state_dict`. Metadata saved for this\n",
    "    module in input :attr:`state_dict` is provided as :attr:`local_metadata`.\n",
    "    For state dicts without metadata, :attr:`local_metadata` is empty.\n",
    "    Subclasses can achieve class-specific backward compatible loading using\n",
    "    the version number at `local_metadata.get(\"version\", None)`.\n",
    "\n",
    "    .. note::\n",
    "        :attr:`state_dict` is not the same object as the input\n",
    "        :attr:`state_dict` to :meth:`~torch.nn.Module.load_state_dict`. So\n",
    "        it can be modified.\n",
    "\n",
    "    Arguments:\n",
    "        state_dict (dict): a dict containing parameters and\n",
    "            persistent buffers.\n",
    "        prefix (str): the prefix for parameters and buffers used in this\n",
    "            module\n",
    "        local_metadata (dict): a dict containing the metadata for this module.\n",
    "            See\n",
    "        strict (bool): whether to strictly enforce that the keys in\n",
    "            :attr:`state_dict` with :attr:`prefix` match the names of\n",
    "            parameters and buffers in this module\n",
    "        missing_keys (list of str): if ``strict=True``, add missing keys to\n",
    "            this list\n",
    "        unexpected_keys (list of str): if ``strict=True``, add unexpected\n",
    "            keys to this list\n",
    "        error_msgs (list of str): error messages should be added to this\n",
    "            list, and will be reported together in\n",
    "            :meth:`~torch.nn.Module.load_state_dict`\n",
    "    \"\"\"\n",
    "    for hook in self._load_state_dict_pre_hooks.values():\n",
    "        hook(state_dict, prefix, local_metadata, strict, missing_keys,\n",
    "             unexpected_keys, error_msgs)\n",
    "\n",
    "    local_name_params = itertools.chain(self._parameters.items(),\n",
    "                                        self._buffers.items())\n",
    "    local_state = {k: v.data for k, v in local_name_params if v is not None}\n",
    "\n",
    "    for name, param in local_state.items():\n",
    "        key = prefix + name\n",
    "        if key in state_dict:\n",
    "            input_param = state_dict[key]\n",
    "\n",
    "            # Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+\n",
    "            if len(param.shape) == 0 and len(input_param.shape) == 1:\n",
    "                input_param = input_param[0]\n",
    "\n",
    "            if input_param.shape != param.shape:\n",
    "                # local shape should match the one in checkpoint\n",
    "                error_msgs.append(\n",
    "                    'size mismatch for {}: copying a param with shape {} from checkpoint, '\n",
    "                    'the shape in current model is {}.'.format(\n",
    "                        key, input_param.shape, param.shape))\n",
    "                #if not strict:\n",
    "                #    continue\n",
    "\n",
    "            if isinstance(input_param, Parameter):\n",
    "                # backwards compatibility for serialized parameters\n",
    "                input_param = input_param.data\n",
    "\n",
    "            try:\n",
    "                if False:\n",
    "                    input_param = input_param[torch.randperm(input_param.size()[0])]\n",
    "                param.copy_(input_param)\n",
    "            except Exception:\n",
    "                error_msgs.append(\n",
    "                    'While copying the parameter named \"{}\", '\n",
    "                    'whose dimensions in the model are {} and '\n",
    "                    'whose dimensions in the checkpoint are {}.'.format(\n",
    "                        key, param.size(), input_param.size()))\n",
    "                # PG load partially\n",
    "\n",
    "                if len(input_param.size()) == 3:\n",
    "                    error_msgs.append(\n",
    "                        'Partially copying the parameter named \"{}\", '\n",
    "                        'whose dimensions in the model are {} and '\n",
    "                        'whose dimensions in the checkpoint are {}. - trying {}'\n",
    "                        .format(\n",
    "                            key, param.size(), input_param.size(),\n",
    "                            param[:input_param.size()[0], :input_param.size(\n",
    "                            )[1], :input_param.size()[2]].shape))\n",
    "                else:\n",
    "                    error_msgs.append(\n",
    "                        'Partially copying the parameter named \"{}\", '\n",
    "                        'whose dimensions in the model are {} and '\n",
    "                        'whose dimensions in the checkpoint are {}. - trying {}'\n",
    "                        .format(key, param.size(), input_param.size(),\n",
    "                                param[:input_param.size()[0]].shape))\n",
    "\n",
    "                try:\n",
    "                    new_input_param = torch.empty_like(param)\n",
    "                    new_input_param = torch.nn.init.normal_(new_input_param,\n",
    "                                                            mean=input_param.mean(),\n",
    "                                                            std=input_param.std())\n",
    "\n",
    "                    if len(input_param.size()) == 3:\n",
    "                        new_input_param[:input_param.size()[0], :input_param.\n",
    "                                        size()[1], :input_param.size(\n",
    "                                        )[2]] = input_param\n",
    "                    else:\n",
    "                        new_input_param[:input_param.size()[0]] = input_param\n",
    "                    param.copy_(new_input_param)\n",
    "                except Exception as e:\n",
    "                    assert e\n",
    "                    error_msgs.append(\n",
    "                        'Failed to load weights partially {}'.format(e))\n",
    "        elif strict:\n",
    "            missing_keys.append(key)\n",
    "\n",
    "    if strict:\n",
    "        for key in state_dict.keys():\n",
    "            if key.startswith(prefix):\n",
    "                input_name = key[len(prefix):]\n",
    "                input_name = input_name.split(\n",
    "                    '.', 1)[0]  # get the name of param/buffer/child\n",
    "                if input_name not in self._modules and input_name not in local_state:\n",
    "                    unexpected_keys.append(key)\n",
    "\n",
    "def load_state_dict(self, state_dict, strict=True):\n",
    "    r\"\"\"Copies parameters and buffers from :attr:`state_dict` into\n",
    "    this module and its descendants. If :attr:`strict` is ``True``, then\n",
    "    the keys of :attr:`state_dict` must exactly match the keys returned\n",
    "    by this module's :meth:`~torch.nn.Module.state_dict` function.\n",
    "\n",
    "    Arguments:\n",
    "        state_dict (dict): a dict containing parameters and\n",
    "            persistent buffers.\n",
    "        strict (bool, optional): whether to strictly enforce that the keys\n",
    "            in :attr:`state_dict` match the keys returned by this module's\n",
    "            :meth:`~torch.nn.Module.state_dict` function. Default: ``True``\n",
    "\n",
    "    Returns:\n",
    "        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:\n",
    "            * **missing_keys** is a list of str containing the missing keys\n",
    "            * **unexpected_keys** is a list of str containing the unexpected keys\n",
    "    \"\"\"\n",
    "    missing_keys = []\n",
    "    unexpected_keys = []\n",
    "    error_msgs = []\n",
    "\n",
    "    # copy state_dict so _load_from_state_dict can modify it\n",
    "    metadata = getattr(state_dict, '_metadata', None)\n",
    "    state_dict = state_dict.copy()\n",
    "    if metadata is not None:\n",
    "        state_dict._metadata = metadata\n",
    "\n",
    "    def load(module, prefix=''):\n",
    "        local_metadata = {} if metadata is None else metadata.get(\n",
    "            prefix[:-1], {})\n",
    "        module._load_from_state_dict(state_dict, prefix, local_metadata, True,\n",
    "                                     missing_keys, unexpected_keys, error_msgs)\n",
    "        for name, child in module._modules.items():\n",
    "            if child is not None:\n",
    "                load(child, prefix + name + '.')\n",
    "                \n",
    "    load(self)\n",
    "\n",
    "    if strict:\n",
    "        if len(unexpected_keys) > 0:\n",
    "            error_msgs.insert(\n",
    "                0, 'Unexpected key(s) in state_dict: {}. '.format(', '.join(\n",
    "                    '\"{}\"'.format(k) for k in unexpected_keys)))\n",
    "        if len(missing_keys) > 0:\n",
    "            error_msgs.insert(\n",
    "                0, 'Missing key(s) in state_dict: {}. '.format(', '.join(\n",
    "                    '\"{}\"'.format(k) for k in missing_keys)))\n",
    "\n",
    "    if strict and len(error_msgs) > 0:\n",
    "        raise RuntimeError(\n",
    "            'Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n",
    "                self.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:39.233169Z",
     "start_time": "2019-08-19T17:09:39.227819Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nn.Module._load_from_state_dict = _load_from_state_dict\n",
    "nn.Module.load_state_dict = load_state_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:41.060744Z",
     "start_time": "2019-08-19T17:09:39.851184Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.basic_train import *\n",
    "from fastai.basic_data import *\n",
    "from fastai.data_block import *\n",
    "from fastai.torch_core import *\n",
    "from fastai.train import *\n",
    "from fastai.callback import *\n",
    "from fastai.callbacks import *\n",
    "from fastai.distributed import *\n",
    "from fastai.layers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:41.101982Z",
     "start_time": "2019-08-19T17:09:41.063665Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fname_ext = lambda fname,ext: f'{str(fname)[:-4]}{ext}{str(fname)[-4:]}'\n",
    "\n",
    "def preprocess(fname, type_index=None, ext=''):\n",
    "    t  = pd.read_csv(fname_ext(fname,ext))\n",
    "    s  = pd.read_csv('structures.csv')\n",
    "    \n",
    "    has_y = 'scalar_coupling_constant' in t.columns\n",
    "\n",
    "    if has_y:\n",
    "        # atom-atom level\n",
    "        # molecule_name,atom_index_0,atom_index_1,type,fc,sd,pso,dso\n",
    "        scalar_couplings = pd.read_csv(f'scalar_coupling_contributions{ext}.csv') # fc,sd,pso,dso\n",
    "\n",
    "        # atom level\n",
    "        # molecule_name,atom_index,XX,YX,ZX,XY,YY,ZY,XZ,YZ,ZZ\n",
    "        magnetic_shielding = pd.read_csv('magnetic_shielding_tensors.csv')\n",
    "        # molecule_name,atom_index,mulliken_charge\n",
    "        mulliken_charges = pd.read_csv('mulliken_charges.csv')\n",
    "\n",
    "        # molecule level\n",
    "        # molecule_name,X,Y,Z\n",
    "        dipole_moments = pd.read_csv('dipole_moments.csv')\n",
    "        # molecule_name,potential_energy\n",
    "        potential_energy = pd.read_csv('potential_energy.csv')\n",
    "\n",
    "    t['molecule_index'] = pd.factorize(t['molecule_name'])[0] + t['id'].min()\n",
    "    # make sure we use the same indexes in train/test (test needs to provide type_index)\n",
    "    if type_index is not None:\n",
    "        t['type_idx'] = t['type'].apply(lambda x: type_index.index(x) ) # pd.factorize(pd.concat([pd.Series(type_index),t['type']]))[0][len(type_index):]\n",
    "    else:\n",
    "        t['type_idx'] = pd.factorize(t['type'])[0]\n",
    "\n",
    "    s['atom_idx'] = s['atom'].apply(lambda x: atoms.index(x) )\n",
    "\n",
    "    max_items = len(t.groupby(['molecule_name', 'atom_index_0']))# if has_y else 422550\n",
    "    max_atoms = int(s.atom_index.max() + 1)\n",
    "\n",
    "    if has_y:\n",
    "        contributions = ['fc','sd','pso','dso']\n",
    "        magnetic_tensors = ['XX','YX','ZX','XY','YY','ZY','XZ','YZ','ZZ']\n",
    "        XYZ = ['X','Y','Z']\n",
    "    xyz = ['x', 'y', 'z']\n",
    "    \n",
    "    x_xyz   = np.zeros((max_items,len(xyz),  max_atoms), dtype=np.float32)\n",
    "    x_type  = np.zeros((max_items,1,         max_atoms), dtype=np.int8)\n",
    "    x_ext   = np.zeros((max_items,1,         max_atoms), dtype=np.bool_)\n",
    "    x_atom  = np.empty((max_items,1,         max_atoms), dtype=np.int8)\n",
    "    x_atom[:] = -1\n",
    "\n",
    "    if has_y:\n",
    "        y_scalar   = np.zeros((max_items,len(contributions)   ,max_atoms), dtype=np.float32)\n",
    "        y_magnetic = np.zeros((max_items,len(magnetic_tensors),max_atoms), dtype=np.float32)\n",
    "        y_mulliken = np.zeros((max_items,1                    ,max_atoms), dtype=np.float32)\n",
    "\n",
    "        y_dipole   = np.zeros((max_items,len(XYZ)), dtype=np.float32)\n",
    "        y_potential= np.zeros((max_items,1       ), dtype=np.float32)\n",
    "\n",
    "        y_magnetic[...] = np.nan\n",
    "        y_mulliken[...] = np.nan\n",
    "    else:\n",
    "        xt_ids = np.zeros((max_items, max_atoms), dtype=np.int32)\n",
    "\n",
    "    m = np.zeros((max_items,), dtype=np.int32)\n",
    "    i = j = 0\n",
    "    \n",
    "    for (m_name, m_index) ,m_group in tqdm(t.groupby(['molecule_name', 'molecule_index'])):\n",
    "        ss = s[s.molecule_name==m_name]\n",
    "        n_atoms = len(ss)\n",
    "        if has_y:\n",
    "            magnetic = magnetic_shielding[\n",
    "                    (magnetic_shielding['molecule_name']==m_name)][magnetic_tensors].values.T\n",
    "\n",
    "            mulliken = mulliken_charges[\n",
    "                    (mulliken_charges['molecule_name']==m_name)]['mulliken_charge'].values.T\n",
    "\n",
    "            scs = scalar_couplings[scalar_couplings['molecule_name']==m_name]\n",
    "            \n",
    "            y_dipole[j,:]= dipole_moments[dipole_moments['molecule_name']==m_name][XYZ].values\n",
    "            y_potential[j,:]=potential_energy[\n",
    "                potential_energy['molecule_name']==m_name]['potential_energy'].values\n",
    "        \n",
    "        for a_name,a_group in m_group.groupby('atom_index_0'):\n",
    "            \n",
    "            ref_a = ss[ss['atom_index']==a_name]\n",
    "            \n",
    "            x_xyz[i] = 0.\n",
    "            x_type[i] = -1\n",
    "            x_ext[i] =  True\n",
    "            \n",
    "            x_xyz[i,:,:n_atoms] = (ss[xyz].values-ref_a[xyz].values).T  # xyz \n",
    "            x_type[i,0,a_group['atom_index_1']] = a_group['type_idx']  # type \n",
    "            x_ext[i,0,a_group['atom_index_1']] = a_group['ext']  # ext \n",
    "            x_atom[i,:,:n_atoms] = ss['atom_idx'].T                \n",
    "\n",
    "            if has_y:\n",
    "                y_scalar[i,:,a_group['atom_index_1']] = scs[scs['atom_index_0']==a_name][contributions]\n",
    "                y_magnetic[i,:,:n_atoms] = magnetic\n",
    "                y_mulliken[i,:,:n_atoms] = mulliken\n",
    "            else:\n",
    "                xt_ids[i,a_group['atom_index_1']] = a_group['id']  \n",
    "\n",
    "            m[i] = m_index\n",
    "            i+=1\n",
    "        j += 1\n",
    "    assert i == max_items\n",
    "    print(i,max_items)\n",
    "    if has_y:\n",
    "        return x_xyz,x_type,x_ext,x_atom, m, y_scalar, y_magnetic, y_mulliken, y_dipole, y_potential\n",
    "    else:\n",
    "        return x_xyz,x_type,x_ext,x_atom, m, xt_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define where you want to use original training set '' or extended ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:41.692190Z",
     "start_time": "2019-08-19T17:09:41.688409Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ext = '_ext' # or ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load preprocessed or preprocess and save for later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:48.324228Z",
     "start_time": "2019-08-19T17:09:42.786721Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_fname = Path('train.npz')\n",
    "types = ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN'] \n",
    "atoms = 'CFHNO'\n",
    "\n",
    "try:\n",
    "    npzfile = np.load(fname_ext(train_fname, ext))\n",
    "    x_xyz   = npzfile['x_xyz']\n",
    "    x_type  = npzfile['x_type']\n",
    "    x_ext   = npzfile['x_ext']\n",
    "    x_atom  = npzfile['x_atom']\n",
    "\n",
    "    y_scalar    = npzfile['y_scalar']\n",
    "    y_magnetic  = npzfile['y_magnetic']\n",
    "    y_mulliken  = npzfile['y_mulliken']\n",
    "    y_dipole    = npzfile['y_dipole']\n",
    "    y_potential = npzfile['y_potential']\n",
    "    m = npzfile['m']\n",
    "    max_items, max_atoms = x_xyz.shape[0], x_xyz.shape[-1]\n",
    "except:\n",
    "    x_xyz,x_type,x_ext,x_atom, m, y_scalar, y_magnetic, y_mulliken, y_dipole, y_potential = \\\n",
    "        preprocess(train_fname.with_suffix('.csv'), type_index=types, ext=ext)\n",
    "    np.savez(fname_ext(train_fname, ext), \n",
    "             x_xyz=x_xyz,\n",
    "             x_type=x_type,\n",
    "             x_ext=x_ext,\n",
    "             x_atom=x_atom,\n",
    "             y_scalar=y_scalar,\n",
    "             y_magnetic=y_magnetic,\n",
    "             y_mulliken=y_mulliken,\n",
    "             y_dipole=y_dipole,\n",
    "             y_potential=y_potential,\n",
    "             m=m)\n",
    "n_types = int(x_type[~np.isnan(x_type)].max() + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:48.332322Z",
     "start_time": "2019-08-19T17:09:48.326293Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "use_memmap = True\n",
    "try:\n",
    "    load_fn = np.load if not use_memmap else partial(np.lib.format.open_memmap, mode='r')\n",
    "    x_coulombmat = load_fn(f'x_coulombmat32{ext}.npy')\n",
    "except:\n",
    "    x_coulombmat = np.load(f'x_coulombmat{ext}.npy', allow_pickle=True)\n",
    "    x_coulombmat = np.array(x_coulombmat.tolist()).astype(np.float32)\n",
    "    np.save(f'x_coulombmat32{ext}.npy', x_coulombmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:48.338202Z",
     "start_time": "2019-08-19T17:09:48.334641Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_qm9_mulliken = load_fn(f'x_qm9_mulliken{ext}.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:48.346751Z",
     "start_time": "2019-08-19T17:09:48.340324Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1405126, 3, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 4, 29),\n",
       " (1405126, 9, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 3),\n",
       " (1405126, 1),\n",
       " (1405126,)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.shape for v in [x_xyz,x_type,x_ext,x_atom,x_qm9_mulliken, \n",
    "                   y_scalar, y_magnetic, y_mulliken, y_dipole, y_potential, m]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.367867Z",
     "start_time": "2019-08-19T17:09:48.348694Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_xyz_mean, x_xyz_std = Tensor(x_xyz.mean(axis=(0,2),keepdims=True)), Tensor(x_xyz.std(axis=(0,2),keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.647878Z",
     "start_time": "2019-08-19T17:09:49.369847Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_qm9_mulliken_mean = Tensor(x_qm9_mulliken.mean(axis=(0,2),keepdims=True))\n",
    "x_qm9_mulliken_std  = Tensor(x_qm9_mulliken.std( axis=(0,2),keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.653815Z",
     "start_time": "2019-08-19T17:09:49.650096Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 1])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_xyz_std.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fastai classes (this should should be done into its own `application` but who has time?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.673551Z",
     "start_time": "2019-08-19T17:09:49.657553Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MoleculeItem(ItemBase):\n",
    "    def __init__(self,i,xyz,type,ext,atom,qm9_mulliken,coulomb): \n",
    "        self.i,self.xyz,self.type,self.ext, self.atom,self.qm9_mulliken,self.coulomb = \\\n",
    "            i,xyz,type,ext,atom,qm9_mulliken,coulomb\n",
    "        self.data = [Tensor(xyz), LongTensor((type)), \n",
    "                     Tensor(ext), LongTensor((atom)),Tensor(qm9_mulliken), Tensor(coulomb)]\n",
    "    def __str__(self):\n",
    "        # TODO: count n_atoms correctly. \n",
    "        n_atoms = np.count_nonzero(np.sum(np.absolute(self.xyz), axis=0))+1\n",
    "        n_couplings = np.sum((self.type!=-1))\n",
    "        return f'{self.i} {n_atoms} atoms {n_couplings} couplings'\n",
    "    \n",
    "    def apply_tfms(self, tfms:Collection, **kwargs):\n",
    "        x = self.clone()\n",
    "        for t in tfms:\n",
    "            if t: x.data = t(x.data)\n",
    "        return x\n",
    "    \n",
    "    def clone(self):\n",
    "        return self.__class__(self.i,self.xyz,self.type,self.ext,self.atom,self.qm9_mulliken,self.coulomb)\n",
    "    \n",
    "class ScalarCouplingItem(ItemBase):\n",
    "    def __init__(self,scalar,magnetic,mulliken,dipole,potential,**kwargs): \n",
    "        self.scalar,self.magnetic,self.mulliken,self.dipole,self.potential = \\\n",
    "            scalar,magnetic,mulliken,dipole,potential\n",
    "        self.data = (Tensor(scalar), Tensor(magnetic), Tensor(dipole), Tensor(potential))\n",
    "    def __str__(self):\n",
    "        res, spacer, n_couplings = '', '', 0\n",
    "        for s in self.data[0].sum(dim=0):\n",
    "            if s==0.: spacer = ' * '\n",
    "            else: \n",
    "                res += f'{spacer}{s:.4f}'\n",
    "                spacer = ' '\n",
    "                n_couplings +=1\n",
    "        return f'{n_couplings}: {res}'\n",
    "    def __hash__(self): return hash(str(self))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.691659Z",
     "start_time": "2019-08-19T17:09:49.675862Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LMAEMaskedLoss(Module):\n",
    "    def __init__(self,\n",
    "                 contrib_w=0., magnetic_w=0., dipole_w=0., potential_w=0., \n",
    "                 types_w = [1]*n_types, return_all=False, proxy_log=torch.log, exclude_ext=False):\n",
    "        self.contrib_w,self.magnetic_w,self.dipole_w,self.potential_w = contrib_w,magnetic_w,dipole_w,potential_w\n",
    "        self.types_w = types_w\n",
    "        self.return_all = return_all\n",
    "        self.proxy_log = proxy_log\n",
    "        self.exclude_ext = exclude_ext\n",
    "    \n",
    "    def forward(self, input_outputs, t_scalar, t_magnetic, t_dipole, t_potential):    \n",
    "        type, ext, p_scalar, p_magnetic, p_dipole, p_potential = input_outputs\n",
    "        loss = 0.\n",
    "        n = 0\n",
    "        j_loss = [0] * n_types\n",
    "        for t in range(n_types):\n",
    "            mask = (type == t).squeeze(1) if not self.exclude_ext else ((type == t) & (ext == 0)).squeeze(1)\n",
    "            if mask.sum() > 0:\n",
    "                _output,_target = p_scalar.transpose(1,2)[mask], t_scalar.transpose(1,2)[mask] # scalars at the end\n",
    "                # LMAE scalar\n",
    "                s_loss = self.proxy_log((_output.sum(dim=-1) - _target.sum(dim=-1)).abs().mean()+1e-9)\n",
    "                loss += self.types_w[t] * s_loss\n",
    "                j_loss[t] += s_loss\n",
    "                # LMAE scalar contributions\n",
    "                #for i_contrib in range(_output.shape[-1]):\n",
    "                #    loss += self.contrib_w * \\\n",
    "                #        self.proxy_log((_output[...,i_contrib] - _target[...,i_contrib]).abs().mean()+1e-9)\n",
    "                n+=1\n",
    "        loss /= n\n",
    "        \n",
    "        if self.magnetic_w > 0:\n",
    "            mask = ~torch.isnan(t_magnetic)\n",
    "            loss += self.magnetic_w * MSELossFlat()(p_magnetic[mask], t_magnetic[mask])\n",
    "            \n",
    "        if self.dipole_w    > 0: loss += self.dipole_w    * MSELossFlat()(p_dipole,    t_dipole)\n",
    "        if self.potential_w > 0: loss += self.potential_w * MSELossFlat()(p_potential, t_potential)\n",
    "\n",
    "        return loss if not self.return_all else (loss, *j_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.698750Z",
     "start_time": "2019-08-19T17:09:49.693805Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ScalarCouplingList(ItemList):\n",
    "    def __init__(self, items:Iterator, **kwargs):\n",
    "        super().__init__(items, **kwargs)\n",
    "        self.loss_func = LMAEMaskedLoss\n",
    "\n",
    "    def get(self, i):\n",
    "        o = super().get(i)\n",
    "        return ScalarCouplingItem(*o)\n",
    "\n",
    "    def reconstruct(self,t): return 0; # TODO for viz !!!! ScalarCouplingItem(t.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Quaterions allow us to rotate 3d points randoming with a nice uniform distribution of 3 numbers hece we use them, however it's still to be seen if are useful here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:09:49.733919Z",
     "start_time": "2019-08-19T17:09:49.700715Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#https://github.com/facebookresearch/QuaterNet/blob/master/common/quaternion.py\n",
    "def qrot(q, v):\n",
    "    \"\"\"\n",
    "    Rotate vector(s) v about the rotation described by quaternion(s) q.\n",
    "    Expects a tensor of shape (*, 4) for q and a tensor of shape (*, 3) for v,\n",
    "    where * denotes any number of dimensions.\n",
    "    Returns a tensor of shape (*, 3).\n",
    "    \"\"\"\n",
    "    assert q.shape[-1] == 4\n",
    "    assert v.shape[-1] == 3\n",
    "    assert q.shape[:-1] == v.shape[:-1]\n",
    "    \n",
    "    original_shape = list(v.shape)\n",
    "    q = q.view(-1, 4)\n",
    "    v = v.view(-1, 3)\n",
    "    \n",
    "    qvec = q[:, 1:]\n",
    "    uv = torch.cross(qvec, v, dim=1)\n",
    "    uuv = torch.cross(qvec, uv, dim=1)\n",
    "    return (v + 2 * (q[:, :1] * uv + uuv)).view(original_shape)\n",
    "\n",
    "def random_rotation(data):\n",
    "    x_xyz = data[0].transpose(0,1)\n",
    "    r = torch.rand(3)\n",
    "    sq1_v1,sqv1,v2_2pi,v3_2pi = torch.sqrt(1-r[:1]),torch.sqrt(r[:1]),2*math.pi*r[1:2],2*math.pi*r[2:3]\n",
    "    q = torch.cat([sq1_v1*torch.sin(v2_2pi), sq1_v1*torch.cos(v2_2pi), \n",
    "                   sqv1  *torch.sin(v3_2pi), sqv1  *torch.cos(v3_2pi)], dim=0).unsqueeze(0)\n",
    "    x_xyz = qrot(q.expand(x_xyz.shape[0],-1), x_xyz).squeeze(0).transpose(0,1)\n",
    "    return (x_xyz, *data[1:])\n",
    "\n",
    "def normalize(data):\n",
    "    sq = False\n",
    "    if data[0].ndim < 3:\n",
    "        data[0].unsqueeze_(0)\n",
    "        data[4].unsqueeze_(0)\n",
    "        sq = True\n",
    "    x_xyz      = (data[0] - x_xyz_mean)          / x_xyz_std\n",
    "    x_mulliken = (data[4] - x_qm9_mulliken_mean) / x_qm9_mulliken_std\n",
    "    if sq:\n",
    "        x_xyz.squeeze_(0)\n",
    "        x_mulliken.squeeze_(0)\n",
    "    return (x_xyz, data[1],data[2],data[3],x_mulliken,data[5])\n",
    "\n",
    "def canonize_(data):\n",
    "    xyz,type,ext,atom,mulliken,coulomb = data\n",
    "    mask = (atom == -1).squeeze(0)\n",
    "    mask_atoms = ~mask.unsqueeze(0)\n",
    "    n_atoms = mask_atoms.sum()\n",
    "    i = torch.nonzero(type.squeeze(0) == -1)[0] # pick first one w/o j-coupling\n",
    "    xyz[:,mask], type[:,mask],ext[:,mask],atom[:,mask],mulliken[:,mask] = 0,-1,1,-1,0\n",
    "    return (xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms)\n",
    "\n",
    "def canonize(data):\n",
    "    xyz,type,ext,atom,mulliken,coulomb = data\n",
    "    mask = (atom == -1).squeeze(0)\n",
    "    i_max_atom = torch.nonzero(atom != -1).max() + 1\n",
    "    mask_atoms = torch.ones ((max_atoms,  max_atoms), dtype=torch.uint8)\n",
    "    zeros      = torch.zeros((i_max_atom,i_max_atom), dtype=torch.uint8)\n",
    "    mask_atoms[:zeros.shape[0],:zeros.shape[1]] = zeros\n",
    "    n_atoms = i_max_atom\n",
    "    xyz[:,mask], type[:,mask],ext[:,mask],atom[:,mask],mulliken[:,mask] = 0,-1,1,-1,0\n",
    "    return (xyz,type,ext,atom,mulliken, coulomb, mask_atoms, n_atoms)\n",
    "\n",
    "def knockout(data):\n",
    "    xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms = data\n",
    "    atom_to_knockout = random.randint(0, n_atoms-1)\n",
    "    mask_atoms[atom_to_knockout,:] = 1\n",
    "    return xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms\n",
    "\n",
    "def canonize2(data):\n",
    "    xyz,type,ext,atom,mulliken = data\n",
    "    mask = (atom == -1).squeeze(0)\n",
    "    i_max_atom = torch.nonzero(atom != -1).max() + 1\n",
    "    #mask_atoms = torch.ones((max_atoms,   max_atoms), dtype=torch.uint8)\n",
    "    #zeros      = torch.zeros((i_max_atom, i_max_atom), dtype=torch.uint8)\n",
    "    #mask_atoms[:zeros.shape[0],:zeros.shape[1]] = zeros\n",
    "\n",
    "    mask_atoms = torch.ones ((max_atoms, ), dtype=torch.uint8)\n",
    "    zeros      = torch.zeros((i_max_atom,), dtype=torch.uint8)\n",
    "    mask_atoms[:zeros.shape[0],] = zeros\n",
    "    #mask_atoms[(type==-1).squeeze(0),:] = 1\n",
    "    n_atoms = i_max_atom\n",
    "    xyz[:,mask], type[:,mask],ext[:,mask],atom[:,mask],mulliken[:,mask] = 0,-1,1,-1,0\n",
    "    return (xyz,type,ext,atom,mulliken, mask_atoms, n_atoms)\n",
    "\n",
    "def canonize_and_shuffle(data):\n",
    "    xyz,type,ext,atom,mulliken,coulomb = data\n",
    "    mask = (atom == -1).squeeze(0)\n",
    "    mask_atoms = ~mask.unsqueeze(0)\n",
    "    n_atoms = mask_atoms.sum()\n",
    "    i = torch.nonzero(type.squeeze(0) == -1)[0] # pick first one w/o j-coupling\n",
    "    xyz[:,mask], type[:,mask],ext[:,mask],atom[:,mask],mulliken[:,mask] = \\\n",
    "        0,-1,1,-1,0\n",
    "    \n",
    "    perm = torch.randperm(n_atoms)\n",
    "    xyz[:,~mask] = xyz[:,~mask][:,perm]\n",
    "    type[:,~mask] = type[:,~mask][:,perm]\n",
    "    ext[:,~mask] = ext[:,~mask][:,perm]\n",
    "    atom[:,~mask] = atom[:,~mask][:,perm]\n",
    "    mulliken[:,~mask] = mulliken[:,~mask][:,perm]\n",
    "    \n",
    "    return (xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build `data` bunch etc. for fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:18.654986Z",
     "start_time": "2019-08-19T17:09:49.735995Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = ItemList(items=(MoleculeItem(i,*v) for i,v in \n",
    "                       enumerate(zip(x_xyz,x_type,x_ext,x_atom,x_qm9_mulliken,x_coulombmat))),\n",
    "                label_cls=ScalarCouplingItem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:19.035949Z",
     "start_time": "2019-08-19T17:11:18.657069Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "_, idx_valid_split = train_test_split(range(m.max()+1), test_size=0.1, random_state=13)\n",
    "idx_valid_split = np.argwhere(np.isin(m, idx_valid_split)).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:19.162729Z",
     "start_time": "2019-08-19T17:11:19.038706Z"
    },
    "collapsed": true,
    "pixiedust": {
     "displayParams": {}
    }
   },
   "outputs": [],
   "source": [
    "data = data.split_by_idx(idx_valid_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:22.994242Z",
     "start_time": "2019-08-19T17:11:19.164755Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = data.label_from_func(\n",
    "    func=lambda o: (y_scalar[o.i], y_magnetic[o.i], y_mulliken[o.i], y_dipole[o.i], y_potential[o.i]),\n",
    "    label_cls=ScalarCouplingList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.010259Z",
     "start_time": "2019-08-19T17:11:23.002572Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<function normalize at 0x7f8b625362f0>, <function canonize at 0x7f8b60d5f9d8>] [<function normalize at 0x7f8b625362f0>, <function canonize at 0x7f8b60d5f9d8>]\n"
     ]
    }
   ],
   "source": [
    "tfms = [normalize, canonize]\n",
    "tta_tfms = list(tfms)\n",
    "#tta_tfms.insert(0,random_rotation)\n",
    "#tta_tfms.append(knockout)\n",
    "#tta_tfms[-1] = canonize_and_shuffle\n",
    "print(tta_tfms, tfms)\n",
    "data = data.transform((tta_tfms, tfms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.143873Z",
     "start_time": "2019-08-19T17:11:23.012470Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data=data.databunch()#num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelList (78217 items)\n",
       "x: ItemList\n",
       "16 8 atoms 7 couplings,17 8 atoms 7 couplings,18 8 atoms 7 couplings,19 8 atoms 7 couplings,20 8 atoms 7 couplings\n",
       "y: ScalarCouplingList\n",
       "7: 83.5430 -2.3783 * -11.7004 -11.6979 3.2528 13.6913 3.2521,7: 83.5417 -2.3786 -11.7004 * -11.6996 13.6924 3.2525 3.2527,7: 83.5484 -2.3772 -11.6979 -11.6996 * 3.2524 3.2524 13.6921,7: -2.3788 83.5418 3.2528 13.6924 3.2524 * -11.7004 -11.6993,7: -2.3785 83.5430 13.6913 3.2525 3.2524 -11.7004 * -11.6976\n",
       "Path: ."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train_ds.filter_by_func(lambda item, _: len(item.data[2].cpu().numpy()[item.data[2].cpu().numpy()==0]) == 0)\n",
    "data.valid_ds.filter_by_func(lambda item, _: len(item.data[2].cpu().numpy()[item.data[2].cpu().numpy()==0]) == 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "Whole model here, self-contained (needs some cleanup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.162815Z",
     "start_time": "2019-08-19T17:11:23.146818Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LMAEMetric(LearnerCallback):\n",
    "    _order=-20 # Needs to run before the recorder\n",
    "    def __init__(self, learn, val_only=True):\n",
    "        super().__init__(learn)\n",
    "        self.val_only=val_only\n",
    "        self.metric = LMAEMaskedLoss(return_all=True, exclude_ext=True)\n",
    "\n",
    "    def on_train_begin(self, **kwargs):\n",
    "        if not self.val_only: self.learn.recorder.add_metric_names(['tLMAE'])\n",
    "        self.learn.recorder.add_metric_names(['👉🏻LMAE👈🏻'] + [f'lmae{i}' for i in range(n_types)])\n",
    "            \n",
    "    def on_batch_end(self, train, last_output, last_target, **kwargs):\n",
    "        if self.val_only and train: return \n",
    "        preds,targs = self.preds[int(train)], self.targs[int(train)] # 0 val 1 train\n",
    "        if preds is None:\n",
    "            targs, preds = listify(last_target), listify(last_output)\n",
    "            targs,preds = [t.detach() for t in targs],[t.detach() for t in preds]\n",
    "        else:\n",
    "            for i,(o,t) in enumerate(zip(last_output, last_target)):\n",
    "                preds[i] = torch.cat([preds[i], o.detach()], dim=0)\n",
    "                targs[i] = torch.cat([targs[i], t.detach()], dim=0)\n",
    "        self.preds[int(train)], self.targs[int(train)] = preds,targs\n",
    "        \n",
    "    def on_epoch_begin(self, **kwargs):\n",
    "        self.targs, self.preds = [None, None], [None, None]\n",
    "\n",
    "    def on_epoch_end(self, last_metrics, **kwargs):\n",
    "        mets = []\n",
    "        if self.preds[1]: mets.append(self.metric.forward(self.preds[1], *self.targs[1])[0]) # just tLMAE\n",
    "        if self.preds[0]: mets.extend(self.metric.forward(self.preds[0], *self.targs[0]))\n",
    "        return add_metrics(last_metrics, mets) if mets else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.240391Z",
     "start_time": "2019-08-19T17:11:23.166832Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Activation = Enum('Activation', 'ReLU Swish GeLU')\n",
    "\n",
    "class PositionalEncoding(Module):\n",
    "    \"Implement the PE function.\"\n",
    "    def __init__(self, d_model, dropout, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "        # Compute the positional encodings once in log space.\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        \n",
    "        position = torch.arange(0., max_len).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0., d_model, 2) * -(math.log(10000.0) / d_model))\n",
    "\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)[:, :-1]\n",
    "        pe = pe.unsqueeze(0)\n",
    "        self.register_buffer('pe', pe)\n",
    "        print(pe.size())\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.transpose(2, 1)\n",
    "        x = x + Variable(self.pe[:, :x.size(1)], \n",
    "                         requires_grad=False)\n",
    "        return self.dropout(x).transpose(2, 1)\n",
    "    \n",
    "class GeLU(Module):\n",
    "    def forward(self, x): return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))\n",
    "\n",
    "class Swish(Module):\n",
    "    def forward(self, x): return x * torch.sigmoid(x)\n",
    "\n",
    "_activ_func = {Activation.ReLU:nn.ReLU(inplace=True), Activation.GeLU:GeLU(), Activation.Swish: Swish()}\n",
    "\n",
    "def feed_forward(d_model:int, d_ff:int, ff_p:float=0., act:Activation=Activation.ReLU, double_drop:bool=True):\n",
    "    layers = [nn.Linear(d_model, d_ff), _activ_func[act]]\n",
    "    if double_drop: layers.append(nn.Dropout(ff_p))\n",
    "    return SequentialEx(*layers, nn.Linear(d_ff, d_model), nn.Dropout(ff_p), MergeLayer(), nn.LayerNorm(d_model))\n",
    "\n",
    "class MultiHeadAttention(Module):\n",
    "    \"MutiHeadAttention.\"\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int=None, resid_p:float=0., attn_p:float=0., bias:bool=False,\n",
    "                 scale:bool=True):\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
    "\n",
    "        self.attention1 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention2 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention3 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention4 = nn.Linear(d_model, d_model, bias=bias)\n",
    "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
    "        self.drop_att,self.drop_res = nn.Dropout(attn_p),nn.Dropout(resid_p)\n",
    "        self.ln = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs):\n",
    "        return self.ln(x + self.drop_res(self.out(self._apply_attention(x, mask=mask, **kwargs))))\n",
    "\n",
    "    def _apply_attention(self, x:Tensor, mask:Tensor=None):\n",
    "        if False:\n",
    "            if mask is not None:\n",
    "                x= x.masked_fill(mask[:,:,0].squeeze(1).unsqueeze(-1).repeat(1, 1, x.size(2)),  0.)\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = self.attention1(x), self.attention2(x), self.attention3(x)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        wq,wk,wv = wq.permute(0, 2, 1, 3),wk.permute(0, 2, 3, 1),wv.permute(0, 2, 1, 3)\n",
    "        attn_score = torch.matmul(wq, wk)\n",
    "        if self.scale: attn_score.div_(self.d_head ** 0.5)\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask,  -1.0E9).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
    "        attn_vec = torch.matmul(attn_prob, wv)\n",
    "        return self.attention4(attn_vec.permute(0, 2, 1, 3).contiguous().contiguous().view(bs, x_len, -1))\n",
    "\n",
    "    def _attention_einsum(self, x, mask=None):\n",
    "        # Permute and matmul is a little bit faster but this implementation is more readable\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = torch.chunk(self.attention(x), 3, dim=-1)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        attn_score = torch.einsum('bind,bjnd->bijn', (wq, wk))\n",
    "        if self.scale: attn_score.mul_(1/(self.d_head ** 0.5))\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask, -float('-inf')).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=2))\n",
    "        attn_vec = torch.einsum('bijn,bjnd->bind', (attn_prob, wv))\n",
    "        return attn_vec.contiguous().view(bs, x_len, -1)\n",
    "\n",
    "\n",
    "class DecoderLayer(Module):\n",
    "    \"Basic block of a Transformer model.\"\n",
    "    #Can't use Sequential directly cause more than one input...\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int, d_inner:int, resid_p:float=0., attn_p:float=0., ff_p:float=0.,\n",
    "                 bias:bool=True, scale:bool=True, act:Activation=Activation.ReLU, double_drop:bool=True,\n",
    "                 attn_cls:Callable=MultiHeadAttention):\n",
    "        self.mhra = attn_cls(n_heads, d_model, d_head, resid_p=resid_p, attn_p=attn_p, bias=bias, scale=scale)\n",
    "        self.ff   = feed_forward(d_model, d_inner, ff_p=ff_p, act=act, double_drop=double_drop)\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs): return self.ff(self.mhra(x, mask=mask, **kwargs))\n",
    "\n",
    "class Transformer(Module):\n",
    "    \"Transformer model: https://arxiv.org/abs/1706.03762.\"\n",
    "    def __init__(self, n_layers:int, n_heads:int, d_model:int, d_head:int, d_inner:int,\n",
    "                 resid_p:float=0., attn_p:float=0., ff_p:float=0., embed_p:float=0., bias:bool=True, scale:bool=True,\n",
    "                 act:Activation=Activation.ReLU, double_drop:bool=True, attn_cls:Callable=MultiHeadAttention,\n",
    "                 learned_pos_enc:bool=True, mask:bool=True):\n",
    "        self.mask = mask\n",
    "        #self.encoder = nn.Embedding(vocab_sz, d_model)\n",
    "        #self.pos_enc = nn.Embedding(ctx_len, d_model) if learned_pos_enc else PositionalEncoding(d_model)\n",
    "        #self.pos_enc = PositionalEncoding(29, 0.1)\n",
    "        self.drop_emb = nn.Dropout(embed_p)\n",
    "        self.layers = nn.ModuleList([DecoderLayer(n_heads, d_model, d_head, d_inner, resid_p=resid_p, attn_p=attn_p,\n",
    "                      ff_p=ff_p, bias=bias, scale=scale, act=act, double_drop=double_drop,\n",
    "                      attn_cls=attn_cls) for k in range(n_layers)])\n",
    "\n",
    "    def reset(self): pass\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        #bs, x_len = x.size()\n",
    "        #pos = torch.arange(0, x_len, device=x.device, dtype=x.dtype)\n",
    "        inp = self.drop_emb(x)# + self.pos_enc(pos)[None]) #.mul_(self.d_model ** 0.5)\n",
    "        #if False:\n",
    "        #    inp += self.pos_enc(x)[None]\n",
    "        #mask = None #torch.triu(x.new_ones(x_len, x_len), diagonal=1).byte()[None,None] if self.mask else None\n",
    "        #[None,:,:None] for einsum implementation of attention\n",
    "        for layer in self.layers: inp = layer(inp, mask=mask)\n",
    "        return inp #For the LinearDecoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.280402Z",
     "start_time": "2019-08-19T17:11:23.242589Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttentionOld(Module):\n",
    "    \"MutiHeadAttention.\"\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int=None, resid_p:float=0., attn_p:float=0., bias:bool=True,\n",
    "                 scale:bool=True):\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
    "        self.attention = nn.Linear(d_model, 3 * n_heads * d_head, bias=bias)\n",
    "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
    "        self.drop_att,self.drop_res = nn.Dropout(attn_p),nn.Dropout(resid_p)\n",
    "        self.ln = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs):\n",
    "        return self.ln(x + self.drop_res(self.out(self._apply_attention(x, mask=mask, **kwargs))))\n",
    "\n",
    "    def _apply_attention(self, x:Tensor, mask:Tensor=None):\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = torch.chunk(self.attention(x), 3, dim=-1)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        wq,wk,wv = wq.permute(0, 2, 1, 3),wk.permute(0, 2, 3, 1),wv.permute(0, 2, 1, 3)\n",
    "        attn_score = torch.matmul(wq, wk)\n",
    "        if self.scale: attn_score.div_(self.d_head ** 0.5)\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask,  -1.0E9).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
    "        attn_vec = torch.matmul(attn_prob, wv)\n",
    "        return attn_vec.permute(0, 2, 1, 3).contiguous().contiguous().view(bs, x_len, -1)\n",
    "    \n",
    "class AtomTransformerOld(Module):\n",
    "    def __init__(self,  n_heads,d_model, d_head=None, **kwargs):\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.transformer = Transformer(n_heads=n_heads,d_model=d_model, d_head=d_head, **kwargs)\n",
    "        \n",
    "        #self.scalar    = nn.Conv1d(d_model+ n_types + 1, 4, 1)\n",
    "        self.scalar    = nn.Conv1d(d_model, 4, 1)\n",
    "        self.magnetic  = nn.Conv1d(d_model, 9, 1)\n",
    "        self.dipole    = nn.Linear(d_model*max_atoms, 3)\n",
    "        self.potential = nn.Linear(d_model*max_atoms, 1)\n",
    "        \n",
    "        self.n_atom_embedding = d_model//2\n",
    "        self.n_type_embedding = d_model - self.n_atom_embedding - 6\n",
    "        self.type_embedding = nn.Embedding(len(types)+1,self.n_type_embedding)\n",
    "        self.atom_embedding = nn.Embedding(len(atoms)+1,self.n_atom_embedding)\n",
    "    \n",
    "        #n_pos_encoder = d_model - n_type_embedding - n_atom_embedding\n",
    "        #self.pos_encoder = nn.Sequential(\n",
    "        #    nn.Conv1d(3+1+1,n_pos_encoder, 1), nn.ReLU(), nn.BatchNorm1d(n_pos_encoder),\n",
    "        #)\n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):\n",
    "        bs, _, n_pts = xyz.shape        \n",
    "        t = self.type_embedding((type+1).squeeze(1)) #* math.sqrt(self.n_atom_embedding) #.transpose(1,2)\n",
    "        a = self.atom_embedding((atom+1).squeeze(1)) #* math.sqrt(self.n_type_embedding) #.transpose(1,2)\n",
    "        \n",
    "        x = torch.cat([xyz, mulliken, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)        \n",
    "        #x = self.pos_encoder(x).transpose(1,2)\n",
    "\n",
    "        x = torch.cat([x.transpose(1,2), t, a], dim=-1) \n",
    "\n",
    "        mask = (coulomb == 0).unsqueeze(1)\n",
    "        #mask = torch.triu(x.new_ones(max_atoms, max_atoms), diagonal=1).byte()[None,:,:,None]#[None,None] \n",
    "        #print(mask.shape, mask)\n",
    "        x = self.transformer(x, mask).transpose(1,2).contiguous()\n",
    "        \n",
    "        #t_one_hot = torch.zeros(bs,n_types+1,n_pts,device=type.device,dtype=x.dtype).scatter_(1,type+1, 1.)\n",
    "        #scalar    = self.scalar(torch.cat([x, t_one_hot], dim=1))\n",
    "        scalar    = self.scalar(x)\n",
    "        \n",
    "        magnetic  = self.magnetic(x) \n",
    "        dipole    = self.dipole(x.view(bs,-1))\n",
    "        potential = self.potential(x.view(bs,-1))\n",
    "                \n",
    "        return type,ext,scalar,magnetic,dipole,potential\n",
    "    \n",
    "    def reset(self): pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.352590Z",
     "start_time": "2019-08-19T17:11:23.282680Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AtomTransformer(Module):\n",
    "    def __init__(self,  n_heads,d_model, d_head=None,dropout=0, **kwargs):\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.transformer = Transformer(n_heads=n_heads,d_model=d_model, d_head=d_head, **kwargs)\n",
    "        \n",
    "        self.scalar    = nn.Conv1d(d_model+ n_types + 1, 1, 1)\n",
    "        self.magnetic  = nn.Conv1d(d_model, 9, 1)\n",
    "        self.dipole    = nn.Linear(d_model*max_atoms, 3)\n",
    "        self.potential = nn.Linear(d_model*max_atoms, 1)\n",
    "        \n",
    "        self.n_atom_embedding = d_model//2\n",
    "        self.n_type_embedding = d_model - self.n_atom_embedding - 3 # -3 best model\n",
    "        self.type_embedding = nn.Embedding(len(types)+1,self.n_type_embedding)\n",
    "        self.atom_embedding = nn.Embedding(len(atoms)+1,self.n_atom_embedding)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    \n",
    "        #n_pos_encoder = d_model - n_type_embedding - n_atom_embedding\n",
    "        #self.pos_encoder = nn.Sequential(\n",
    "        #    nn.Conv1d(3+1+1,n_pos_encoder, 1), nn.ReLU(), nn.BatchNorm1d(n_pos_encoder),\n",
    "        #)\n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):\n",
    "        bs, _, n_pts = xyz.shape        \n",
    "        t = self.type_embedding((type+1).squeeze(1)) #* math.sqrt(self.n_atom_embedding) #.transpose(1,2)\n",
    "        a = self.atom_embedding((atom+1).squeeze(1)) #* math.sqrt(self.n_type_embedding) #.transpose(1,2)\n",
    "        \n",
    "        #x = torch.cat([xyz, mulliken, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)               \n",
    "        #x = torch.cat([xyz, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)               \n",
    "        x = xyz \n",
    "        #x = self.pos_encoder(x).transpose(1,2)\n",
    "\n",
    "        x = torch.cat([x.transpose(1,2), t, a], dim=-1) \n",
    "\n",
    "        mask = mask_atoms.to(dtype=torch.bool).unsqueeze(1)\n",
    "        #mask = (coulomb == 0).unsqueeze(1)\n",
    "        #mask = torch.triu(x.new_ones(max_atoms, max_atoms), diagonal=1).byte()[None,:,:,None]#[None,None] \n",
    "        #print(mask.shape, mask)\n",
    "        x = self.transformer(x, mask).transpose(1,2).contiguous()\n",
    "        x = self.dropout(x)\n",
    "        t_one_hot = torch.zeros(bs,n_types+1,n_pts,device=type.device,dtype=x.dtype).scatter_(1,type+1, 1.)\n",
    "        \n",
    "        scalar    = self.scalar(torch.cat([x, t_one_hot], dim=1))\n",
    "        magnetic  = self.magnetic(x) \n",
    "        dipole    = self.dipole(x.view(bs,-1))\n",
    "        potential = self.potential(x.view(bs,-1))\n",
    "                \n",
    "        return type,ext,scalar,magnetic,dipole,potential\n",
    "    \n",
    "    def reset(self): pass\n",
    "    \n",
    "    \n",
    "class AtomTransformer2(Module):\n",
    "    def __init__(self,  n_heads,d_model, d_head=None, **kwargs):\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.transformer = Transformer(n_heads=n_heads,d_model=d_model, d_head=d_head, **kwargs)\n",
    "        \n",
    "        self.scalar    = nn.Conv1d(d_model+ n_types + 1, 4, 1)\n",
    "        self.magnetic  = nn.Conv1d(d_model, 9, 1)\n",
    "        self.dipole    = nn.Linear(d_model*max_atoms, 3)\n",
    "        self.potential = nn.Linear(d_model*max_atoms, 1)\n",
    "        \n",
    "        self.n_atom_embedding = d_model//2\n",
    "        self.n_type_embedding = d_model - self.n_atom_embedding - 6 # -3 best model\n",
    "        self.type_embedding = nn.Embedding(len(types)+1,self.n_type_embedding)\n",
    "        self.atom_embedding = nn.Embedding(len(atoms)+1,self.n_atom_embedding)\n",
    "    \n",
    "        #n_pos_encoder = d_model - n_type_embedding - n_atom_embedding\n",
    "        #self.pos_encoder = nn.Sequential(\n",
    "        #    nn.Conv1d(3+1+1,n_pos_encoder, 1), nn.ReLU(), nn.BatchNorm1d(n_pos_encoder),\n",
    "        #)\n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):\n",
    "        bs, _, n_pts = xyz.shape        \n",
    "        t = self.type_embedding((type+1).squeeze(1)) #* math.sqrt(self.n_atom_embedding) #.transpose(1,2)\n",
    "        a = self.atom_embedding((atom+1).squeeze(1)) #* math.sqrt(self.n_type_embedding) #.transpose(1,2)\n",
    "        \n",
    "        x = torch.cat([xyz, mulliken, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)               \n",
    "        #x = torch.cat([xyz, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)               \n",
    "        #x = xyz \n",
    "        #x = self.pos_encoder(x).transpose(1,2)\n",
    "\n",
    "        x = torch.cat([x.transpose(1,2), t, a], dim=-1) \n",
    "\n",
    "        mask = (coulomb == 0).unsqueeze(1)\n",
    "        #mask = torch.triu(x.new_ones(max_atoms, max_atoms), diagonal=1).byte()[None,:,:,None]#[None,None] \n",
    "        #print(mask.shape, mask)\n",
    "        x = self.transformer(x, mask).transpose(1,2).contiguous()\n",
    "        t_one_hot = torch.zeros(bs,n_types+1,n_pts,device=type.device,dtype=x.dtype).scatter_(1,type+1, 1.)\n",
    "        \n",
    "        scalar    = self.scalar(torch.cat([x, t_one_hot], dim=1))\n",
    "        magnetic  = self.magnetic(x) \n",
    "        dipole    = self.dipole(x.view(bs,-1))\n",
    "        potential = self.potential(x.view(bs,-1))\n",
    "                \n",
    "        return type,ext,scalar,magnetic,dipole,potential\n",
    "    \n",
    "    def reset(self): pass\n",
    "    \n",
    "\n",
    "class MultiHeadAttention3(Module):\n",
    "    \"MutiHeadAttention.\"\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int=None, resid_p:float=0., attn_p:float=0., bias:bool=False,\n",
    "                 scale:bool=False):\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
    "\n",
    "        self.attention1 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention2 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention3 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention4 = nn.Linear(d_model, d_model, bias=bias)\n",
    "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
    "        self.drop_att,self.drop_res = nn.Dropout(attn_p),nn.Dropout(resid_p)\n",
    "        self.ln = nn.LayerNorm(d_model)\n",
    "        \n",
    "        self.scale=False\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs):\n",
    "        return self.ln(x + self.drop_res(self.out(self._apply_attention(x, mask=mask, **kwargs))))\n",
    "\n",
    "    def _apply_attention(self, x:Tensor, mask:Tensor=None):\n",
    "        if False:\n",
    "            if mask is not None:\n",
    "                x= x.masked_fill(mask[:,:,0].squeeze(1).unsqueeze(-1).repeat(1, 1, x.size(2)),  0.)\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = self.attention1(x), self.attention2(x), self.attention3(x)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        wq,wk,wv = wq.permute(0, 2, 1, 3),wk.permute(0, 2, 3, 1),wv.permute(0, 2, 1, 3)\n",
    "        attn_score = torch.matmul(wq, wk)\n",
    "        if self.scale: attn_score.div_(self.d_head ** 0.5)\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask,  -1.0E9).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
    "        attn_vec = torch.matmul(attn_prob, wv)\n",
    "        return self.attention4(attn_vec.permute(0, 2, 1, 3).contiguous().contiguous().view(bs, x_len, -1))\n",
    "\n",
    "    def _attention_einsum(self, x, mask=None):\n",
    "        # Permute and matmul is a little bit faster but this implementation is more readable\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = torch.chunk(self.attention(x), 3, dim=-1)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        attn_score = torch.einsum('bind,bjnd->bijn', (wq, wk))\n",
    "        if self.scale: attn_score.mul_(1/(self.d_head ** 0.5))\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask, -float('-inf')).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=2))\n",
    "        attn_vec = torch.einsum('bijn,bjnd->bind', (attn_prob, wv))\n",
    "        return attn_vec.contiguous().view(bs, x_len, -1)\n",
    "\n",
    "    \n",
    "class AtomTransformer3(Module):\n",
    "    def __init__(self,  n_heads,d_model, d_head=None, **kwargs):\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.transformer = Transformer(n_heads=n_heads,d_model=d_model, d_head=d_head, **kwargs)\n",
    "        \n",
    "        self.scalar    = nn.Conv1d(d_model+ n_types + 1, 4, 1)\n",
    "        self.magnetic  = nn.Conv1d(d_model, 9, 1)\n",
    "        self.dipole    = nn.Linear(d_model*max_atoms, 3)\n",
    "        self.potential = nn.Linear(d_model*max_atoms, 1)\n",
    "        \n",
    "        self.n_atom_embedding = d_model//2\n",
    "        self.n_type_embedding = d_model - self.n_atom_embedding - 3 # -3 best model\n",
    "        self.type_embedding = nn.Embedding(len(types)+1,self.n_type_embedding)\n",
    "        self.atom_embedding = nn.Embedding(len(atoms)+1,self.n_atom_embedding)\n",
    "        \n",
    "        self.dropout = nn.Dropout(0.05)\n",
    "    \n",
    "        #n_pos_encoder = d_model - n_type_embedding - n_atom_embedding\n",
    "        #self.pos_encoder = nn.Sequential(\n",
    "        #    nn.Conv1d(3+1+1,n_pos_encoder, 1), nn.ReLU(), nn.BatchNorm1d(n_pos_encoder),\n",
    "        #)\n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):\n",
    "        bs, _, n_pts = xyz.shape        \n",
    "        t = self.type_embedding((type+1).squeeze(1)) #* math.sqrt(self.n_atom_embedding) #.transpose(1,2)\n",
    "        a = self.atom_embedding((atom+1).squeeze(1)) #* math.sqrt(self.n_type_embedding) #.transpose(1,2)\n",
    "        \n",
    "        #x = torch.cat([xyz, mulliken, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)               \n",
    "        #x = torch.cat([xyz, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)               \n",
    "        x = xyz \n",
    "        #x = self.pos_encoder(x).transpose(1,2)\n",
    "\n",
    "        x = torch.cat([x.transpose(1,2), t, a], dim=-1) \n",
    "\n",
    "        mask = (coulomb == 0).unsqueeze(1)\n",
    "        #mask = torch.triu(x.new_ones(max_atoms, max_atoms), diagonal=1).byte()[None,:,:,None]#[None,None] \n",
    "        #print(mask.shape, mask)\n",
    "        x = self.transformer(x, mask).transpose(1,2).contiguous()\n",
    "        x = self.dropout(x)\n",
    "        t_one_hot = torch.zeros(bs,n_types+1,n_pts,device=type.device,dtype=x.dtype).scatter_(1,type+1, 1.)\n",
    "        \n",
    "        scalar    = self.scalar(torch.cat([x, t_one_hot], dim=1))\n",
    "        magnetic  = self.magnetic(x) \n",
    "        dipole    = self.dipole(x.view(bs,-1))\n",
    "        potential = self.potential(x.view(bs,-1))\n",
    "                \n",
    "        return type,ext,scalar,magnetic,dipole,potential\n",
    "    \n",
    "    def reset(self): pass\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This callback allows to insert multiple stateful (not averaged) metrics in one pass. Addditionally we could add metrics for train if we want to."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model instantiation: where's all your TPUs/GPUs when you need a decent hyperparam sweep?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:23.367362Z",
     "start_time": "2019-08-19T17:11:23.354688Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from fastai.callbacks import *\n",
    "#from fastai.callbacks import SaveModelCallback\n",
    "class SaveModelCustomCallback(TrackerCallback):\n",
    "    \"A `TrackerCallback` that saves the model when monitored quantity is best.\"\n",
    "    def __init__(self, learn:Learner, monitor:str='valid_loss', mode:str='auto', every:str='improvement', name:str='bestmodel'):\n",
    "        super().__init__(learn, monitor=monitor, mode=mode)\n",
    "        self.every,self.name = every,name\n",
    "        if self.every not in ['improvement', 'epoch']:\n",
    "            warn(f'SaveModel every {self.every} is invalid, falling back to \"improvement\".')\n",
    "            self.every = 'improvement'\n",
    "                 \n",
    "    def jump_to_epoch(self, epoch:int)->None:\n",
    "        try: \n",
    "            self.learn.load(f'{self.name}_{epoch-1}', purge=False)\n",
    "            print(f\"Loaded {self.name}_{epoch-1}\")\n",
    "        except: print(f'Model {self.name}_{epoch-1} not found.')\n",
    "\n",
    "    def on_epoch_end(self, epoch:int, **kwargs:Any)->None:\n",
    "        \"Compare the value monitored to its best score and maybe save the model.\"\n",
    "        if self.every==\"epoch\": self.learn.save(f'{self.name}_{epoch}')\n",
    "        else: #every=\"improvement\"\n",
    "            current = self.get_monitor_value()\n",
    "            if current is not None and self.operator(current, self.best):\n",
    "                print(f'Better model found at epoch {epoch} with {self.monitor} value: {current}.')\n",
    "                self.best = current\n",
    "                self.learn.save(f'{self.name}_{epoch}_{self.best}')\n",
    "\n",
    "    def on_train_end(self, **kwargs):\n",
    "        \"Load the best model.\"\n",
    "        if False and self.every==\"improvement\" and (self.learn.path/f'{self.learn.model_dir}/{self.name}.pth').is_file():\n",
    "            self.learn.load(f'{self.name}', purge=False)\n",
    "            \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:31.712227Z",
     "start_time": "2019-08-19T17:11:23.369116Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antor/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:25: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_.\n"
     ]
    }
   ],
   "source": [
    "net, learner = None,None\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "net = AtomTransformer(n_layers=6, n_heads=16,d_model=1024,d_inner=4096, act=Activation.Swish, dropout=0)\n",
    "#net = AtomTransformer3(n_layers=6, n_heads=16,d_model=1024,d_inner=4096, attn_cls=MultiHeadAttention3)\n",
    "#net = AtomTransformer(n_layers=6, n_heads=16,d_model=2048,d_inner=2048)\n",
    "#net = AtomTransformer(n_layers=6, n_heads=8, d_model=512,d_inner=2048)\n",
    "\n",
    "#net = AtomTransformerOld(n_layers=6, n_heads=8,d_model=512,d_inner=2048, attn_cls=MultiHeadAttentionOld)\n",
    "\n",
    "#net = AtomTransformer2(n_layers=6, n_heads=12,d_model=768,d_inner=3072)\n",
    "\n",
    "\n",
    "#net = AtomTransformer(n_layers=6, n_heads=12,d_model=768,d_inner=3072)\n",
    "\n",
    "#net = AtomTransformer(n_layers=6, n_heads=16,d_model=1024,d_inner=4096, attn_cls=MultiHeadAttentionOld)\n",
    "\n",
    "learner = Learner(data,net, loss_func=LMAEMaskedLoss(),)\n",
    "\n",
    "learner.callbacks.append(LMAEMetric(learner))\n",
    "\n",
    "for p in learner.model.parameters():\n",
    "    if p.dim() > 1:\n",
    "        nn.init.xavier_uniform(p)\n",
    "        \n",
    "\n",
    "learner.callbacks.append(SaveModelCustomCallback(learner, monitor='👉🏻LMAE👈🏻', mode='min'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T13:53:26.675699Z",
     "start_time": "2019-08-19T13:53:24.807346Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AtomTransformer\n",
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "Swish                [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "Swish                [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "Swish                [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "Swish                [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "Swish                [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "Swish                [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Conv1d               [1, 29]              1,034      True      \n",
       "______________________________________________________________________\n",
       "Conv1d               [9, 29]              9,225      True      \n",
       "______________________________________________________________________\n",
       "Linear               [3]                  89,091     True      \n",
       "______________________________________________________________________\n",
       "Linear               [1]                  29,697     True      \n",
       "______________________________________________________________________\n",
       "Embedding            [29, 509]            4,581      True      \n",
       "______________________________________________________________________\n",
       "Embedding            [29, 512]            3,072      True      \n",
       "______________________________________________________________________\n",
       "Dropout              [1024, 29]           0          False     \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 82,011,644\n",
       "Total trainable params: 82,011,644\n",
       "Total non-trainable params: 0\n",
       "Optimized with 'torch.optim.adam.Adam', betas=(0.9, 0.99)\n",
       "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
       "Loss function : LMAEMaskedLoss\n",
       "======================================================================\n",
       "Callbacks functions applied \n",
       "    LMAEMetric\n",
       "    SaveModelCustomCallback"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:31.717606Z",
     "start_time": "2019-08-19T17:11:31.714392Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#sub_fname = \"loss-5.9943val-2.7766\" # uncomment or set None to skip loading trained net\n",
    "#sub_fname = \"loss-5.7552val-2.9950\"\n",
    "#sub_fname = \"loss-5.9943val-2.7766\"\n",
    "#sub_fname = \"loss-4.9516val-3.0042\"\n",
    "#sub_fname = \"loss-4.0414val-2.7287\"\n",
    "#sub_fname = \"loss-4.9044val-2.5880\"\n",
    "\n",
    "#sub_fname = \"loss-5.0862val-3.0047\"\n",
    "\n",
    "#sub_fname = 'loss-4.9516val-2.8229'\n",
    "#sub_fname = 'bestmodel_62_-2.2235493659973145'\n",
    "#sub_fname = 'loss-5.6540val-3.0131'\n",
    "#sub_fname = 'bestmodel_6_-0.6834144592285156'\n",
    "#sub_fname = 'loss-5.1336val-3.0759'\n",
    "sub_fname = 'loss-3.5933val-3.0637'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:41.657025Z",
     "start_time": "2019-08-19T17:11:31.721110Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attempting to load: loss-3.5933val-3.0637... Loaded\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    print(f\"Attempting to load: {sub_fname}... \", end=\"\")\n",
    "    learner.load(sub_fname, strict=False,with_opt=False)\n",
    "    print(\"Loaded\")\n",
    "except Exception as e:\n",
    "    print(\"NOT loaded! \", e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T06:40:14.614754Z",
     "start_time": "2019-08-19T06:40:14.609905Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if False:\n",
    "    for name, param in learner.model.named_parameters():\n",
    "\n",
    "        if 'scalar' not in name:\n",
    "            param.requires_grad = False\n",
    "            pass\n",
    "        else:\n",
    "            print(name)\n",
    "            try:\n",
    "                nn.init.xavier_uniform_(param)\n",
    "            except:\n",
    "                pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:41.662778Z",
     "start_time": "2019-08-19T17:11:41.658966Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner = learner.to_parallel() #b/c it NaNs loss (probably would need to change fp16 settings)\n",
    "data.batch_size = int(4096//2)\n",
    "data.batch_size = data.batch_size // 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Real loss func. Need to test different auxiliary tasks weights: `magnetic_w`, `dipole_w`, `potential`, weights of indivial `lmae`s: `types_w` and maybe `input_transform_w` and `feature_transform_w`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:11:41.668633Z",
     "start_time": "2019-08-19T17:11:41.664619Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type_to_tune = 2\n",
    "types_w = [0] * 8\n",
    "types_w[type_to_tune] = 1\n",
    "learner.loss_func = LMAEMaskedLoss(magnetic_w=0, dipole_w=0, potential_w=0, types_w = types_w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T13:55:31.203434Z",
     "start_time": "2019-08-19T13:53:36.687673Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Failed to compute the gradients, there might not be enough points.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAU3ElEQVR4nO3dfbRldX3f8feHGcDwbJhrlmGIA2ZMM7oSkCvV0Fos6hrsChOr0ZnGVqKVZSNxpVq76GpCLamJYivVSqsso5I0ijys2tFS0VCIjyiX8CAMYseJkREaRoIkoATQb//Ye5jDmXNnLjOzz73D7/1a66zZD7999nfO3ed89sM5v52qQpLUrgMWuwBJ0uIyCCSpcQaBJDXOIJCkxhkEktS45YtdwBO1YsWKWrVq1WKXIUn7lRtuuOF7VTUzad5+FwSrVq1ibm5uscuQpP1Kkr+Yb56nhiSpcQaBJDXOIJCkxhkEktQ4g0CSGjdYECT5cJJ7ktw6z/wkeV+SzUluSfLcoWqRJM1vyCOCjwJrdzH/dGB1/zgL+G8D1iJJmsdgQVBVnwf+ahdN1gF/WJ3rgKOSPH2oeiRJky3mNYJjgDtHxrf203aS5Kwkc0nmtm3bNpXiJKkVixkEmTBt4l1yquqiqpqtqtmZmYm/kJYk7aHFDIKtwLEj4yuBuxapFklq1mIGwUbgn/XfHno+cH9V3b2I9UhSkwbrdC7Jx4FTgRVJtgL/DjgQoKo+AFwJvAzYDPwA+PWhapEkzW+wIKiqDbuZX8Cbhlq/JGlh/GWxJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNGzQIkqxNckeSzUnOmTD/Z5Jck+TGJLckedmQ9UiSdjZYECRZBlwInA6sATYkWTPW7LeBS6vqRGA98F+HqkeSNNmQRwQnA5uraktVPQxcAqwba1PAEf3wkcBdA9YjSZpgyCA4BrhzZHxrP23U24HXJNkKXAn85qQnSnJWkrkkc9u2bRuiVklq1pBBkAnTamx8A/DRqloJvAz4oyQ71VRVF1XVbFXNzszMDFCqJLVryCDYChw7Mr6SnU/9vB64FKCqvgI8BVgxYE2SpDFDBsH1wOokxyU5iO5i8MaxNt8BTgNI8vN0QeC5H0maosGCoKoeBc4GrgJup/t20G1JzktyRt/srcAbktwMfBw4s6rGTx9Jkga0fMgnr6or6S4Cj047d2R4E3DKkDVIknbNXxZLUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxg0aBEnWJrkjyeYk58zT5lVJNiW5LcnHhqxHkrSz5UM9cZJlwIXAS4CtwPVJNlbVppE2q4F/A5xSVfcledpQ9UiSJhvyiOBkYHNVbamqh4FLgHVjbd4AXFhV9wFU1T0D1iNJmmDIIDgGuHNkfGs/bdSzgGcl+VKS65KsnfRESc5KMpdkbtu2bQOVK0ltGjIIMmFajY0vB1YDpwIbgA8lOWqnhaouqqrZqpqdmZnZ54VKUsuGDIKtwLEj4yuBuya0+Z9V9UhV/TlwB10wSJKmZMgguB5YneS4JAcB64GNY20+CbwIIMkKulNFWwasSZI0ZrAgqKpHgbOBq4DbgUur6rYk5yU5o292FXBvkk3ANcDbqureoWqSJO0sVeOn7Ze22dnZmpubW+wyJGm/kuSGqpqdNM9fFktS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGLSgIkjwzycH98KlJ3jypKwhJ0v5noUcEVwA/SvKzwB8AxwHeO0CSngQWGgQ/7n8p/HLgP1fVvwSePlxZkqRpWWgQPJJkA/Ba4NP9tAOHKUmSNE0LDYJfB14AvKOq/jzJccB/H64sSdK0LOhWlf3tJd8MkOSpwOFV9c4hC5MkTcdCvzV0bZIjkvwkcDPwkSTvGbY0SdI0LPTU0JFV9dfAPwY+UlUnAS8erixJ0rQsNAiWJ3k68Cp2XCyWJD0JLDQIzqO7icy3qur6JMcD/3e4siRJ07LQi8WXAZeNjG8BXjFUUZKk6VnoxeKVSf5HknuS/GWSK5KsHLo4SdLwFnpq6CN0N57/aeAY4FP9NEnSfm6hQTBTVR+pqkf7x0eBmQHrkiRNyUKD4HtJXpNkWf94DXDvkIVJkqZjoUHwOrqvjv4/4G7glXTdTkiS9nMLCoKq+k5VnVFVM1X1tKr6Fbofl0mS9nN7c4eyt+yzKiRJi2ZvgiD7rApJ0qLZmyCofVaFJGnR7PKXxUn+hskf+AF+YpCKJElTtcsgqKrDp1WIJGlx7M2pIUnSk4BBIEmNMwgkqXGDBkGStUnuSLI5yTm7aPfKJJVkdsh6JEk7GywIkiwDLgROB9YAG5KsmdDucODNwFeHqkWSNL8hjwhOBjZX1Zaqehi4BFg3od3vAucDDw1YiyRpHkMGwTHAnSPjW/tpj0lyInBsVe3yPshJzkoyl2Ru27Zt+75SSWrYkEEwqQuKx36cluQA4ALgrbt7oqq6qKpmq2p2ZsbbIEjSvjRkEGwFjh0ZXwncNTJ+OPAc4Nok3waeD2z0grEkTdeQQXA9sDrJcUkOAtbT3e4SgKq6v6pWVNWqqloFXAecUVVzA9YkSRozWBBU1aPA2cBVwO3ApVV1W5Lzkpwx1HolSU/MLvsa2ltVdSVw5di0c+dpe+qQtUiSJvOXxZLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxgwZBkrVJ7kiyOck5E+a/JcmmJLckuTrJM4asR5K0s8GCIMky4ELgdGANsCHJmrFmNwKzVfULwOXA+UPVI0mabMgjgpOBzVW1paoeBi4B1o02qKprquoH/eh1wMoB65EkTTBkEBwD3DkyvrWfNp/XA/970owkZyWZSzK3bdu2fViiJGnIIMiEaTWxYfIaYBZ496T5VXVRVc1W1ezMzMw+LFGStHzA594KHDsyvhK4a7xRkhcD/xb4B1X1twPWI0maYMgjguuB1UmOS3IQsB7YONogyYnAB4EzquqeAWuRJM1jsCCoqkeBs4GrgNuBS6vqtiTnJTmjb/Zu4DDgsiQ3Jdk4z9NJkgYy5KkhqupK4MqxaeeODL94yPVLknbPXxZLUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktS4QYMgydokdyTZnOScCfMPTvKJfv5Xk6wash5J0s4GC4Iky4ALgdOBNcCGJGvGmr0euK+qfha4AHjXUPVIkiYb8ojgZGBzVW2pqoeBS4B1Y23WARf3w5cDpyXJgDVJksYMGQTHAHeOjG/tp01sU1WPAvcDR48/UZKzkswlmdu2bdtA5UpSm4YMgkl79rUHbaiqi6pqtqpmZ2Zm9klxkqTOkEGwFTh2ZHwlcNd8bZIsB44E/mrAmiRJY4YMguuB1UmOS3IQsB7YONZmI/DafviVwP+pqp2OCCRJw1k+1BNX1aNJzgauApYBH66q25KcB8xV1UbgD4A/SrKZ7khg/VD1SJImGywIAKrqSuDKsWnnjgw/BPzqkDVIknbNXxZLUuMMAklqnEEgSY0zCCSpcdnfvq2ZZBvwF4tdx5gVwPcWu4gxS7EmWJp1LcWaYGnWtRRrgqVZ11Kr6RlVNfEXuftdECxFSeaqanax6xi1FGuCpVnXUqwJlmZdS7EmWJp1LcWa5uOpIUlqnEEgSY0zCPaNixa7gAmWYk2wNOtaijXB0qxrKdYES7OupVjTRF4jkKTGeUQgSY0zCCSpcc0HQZIPJ7knya1PcLlDkvyvJN9IcluSd47MuyDJTf3jm0m+PzLvXUlu7R+vnmZdI21emaSSzPbjL0lyQ5Kv9//+w2nVlOTgJJ9IsjnJV5Os6qevSvLDkdfxA7t4/j2qq1/2HUnuTPLAhHmvSrKpr/ljI9PP76fdnuR9k26vOkRNSd7Y/41uSvLF7fcAT3J0kmuSPJDk/U9gPXtT46uT3NK/DudPmP+4bWxKNV2b5I6RbeZp/fR5349D17Un2/6iqKqmH8ALgecCtz7B5Q4BXtQPHwR8ATh9QrvfpOuCG+AfAZ+j6/X1UGAOOGKadQGHA58HrgNm+2knAj/dDz8H+O60agJ+A/hAP7we+EQ/vGqh69nTuvplnw88HXhgbPpq4Ebgqf340/p/fwn4El3X6suArwCnTqmmI0aGzwA+0w8fCvw94I3A+6ew7R8NfAeY6ccvBk7b1TY2dE39stfubn2j78dp1LUn2/5iPJo/IqiqzzN2V7Qkz0zymX7v+AtJ/s6E5X5QVdf0ww8Df0Z3F7ZxG4CP98NrgD+tqker6kHgZmDtlOv6XeB84KGRZW6squ13j7sNeEqSg6dU0zq6DxKAy4HTJu1h78qe1tUve11V3T1h1huAC6vqvr7dPdsXAZ5C96Y+GDgQ+Mtp1FRVfz0yemhfC1X1YFV9kZG/6ULsRY3HA9+squ03EP8T4BUj83faxqZQ00KNvh8Hr2vobX+fWawEWkoPxvY+gauB1f3w36W7c9qulj8K2AIcPzb9GcDdwLJ+/KV0e5OH0P38fAvw1mnVRbfnf0U/fC0T9p7o7hT3J1Os6VZg5cj8b/WvzSrgQbq98j8F/v7Af8Pxve9P0n2YfYluz3btyLz/CHwfuB94x7Rq6qe9qX+N7tz+XCPzzuQJHBHsaY3AU+luM7uK7uj2CuBTC93GhqhpZH1fB24Cfof+W5Ej8x/3fpxWXU9029+T2vb2MeiNafZHSQ6jO/y/bCScd9o7Hmm/nG4P431VtWVs9nrg8qr6EUBVfTbJ84AvA9voTis8Oo26khwAXED3YTHfMs8G3kUXWIPXtH3yhKZF94b9maq6N8lJwCeTPLsev1e8T+qax3K600On0u3BfSHJc+hC6ufZsVf3uSQvrG6PceiaqKoLgQuT/BPgt9lxq9e9ttAaq+q+JP8C+ATwY7rt+fiFbGND1dT7tar6bpLD6cLpnwJ/ODL/ce/HKdb1RLf96VuM9FlqD0aSHjgCuHtCm2V0exo3AeeNTP8w3R930vPeCPzSLtb7MeBl06gLOJKuA6xv94+HgLvYcZ1gJfBN4JRpvlZ0tzJ9QT+8vK8xE57zWnaxd7k3dfXzxo8IPgCcOTJ+NfA84G3A74xMPxf419OoaWzeAcD9Y9POZC+OCPakxn7+WXRHT7vcxqZc006vBbt5Pw5Z195s+9N4TH2FS/HBzod8XwZ+tR8O8IvzLPcf6PY8Dpgw7+f6N0NGpi0Dju6Hf4Hu0HD5NOsaaXMtO0LgKLrrFa+Y9mtFd6pj9ILZpf3wDDtOqR0PfBf4yX1d10j78SBYC1zcD6+gOxVzNPBqunPiy+muD1wN/PKUalo9MvzLdPf+Hp1/Jnt/amihf8/tF8+fSveh96xdbWND19T/PVb0wwfSnXN/48j8nd6P06hrT7b9xXgsykqX0oPucO1u4BG6856vB44DPkP34bgJOHfCcivpDuNuZ8cewD8fmf924J1jyzylf75NdOedT5h2XSPtHnuT0p1ieHCk/U3b3+hD19S/JpcBm4GvseP86SvoLlzfTHeBbeKH7d7U1S97fr/Mj/t/395PD/CeftmvA+v76cuAD/b/l03Ae6ZY03v71+Qm4Brg2SPLfJvuYuYD/TJrhtr2R5bdvi2vn6fNY9vYFN6PhwI3ALf0r9F7GbkWwIT342J/TjDPtr8YD7uYkKTGNf/1UUlqnUEgSY0zCCSpcQaBJDXOIJCkxhkE2u+N99I5hfV9aHvPn/vguX7U94p5a5JPJTlqN+2PSvIb+2Ld0nZ+fVT7vSQPVNVh+/D5llfVgrr+2Afreqz2JBfTdeb2jl20XwV8uqqeM4361AaPCPSklGQmyRVJru8fp/TTT07y5SQ39v/+XD/9zCSXJfkU8Nkkp/b921/e9yX/x9t7huynb7+XwwPp7h9wc5LrkvxUP/2Z/fj1Sc5b4FHLV4Bj+uUPS3J1kj9Ldw+CdX2bdwLP7I8i3t23fVu/nluS/Pt9+DKqEQaBnqzeC1xQVc+j+5Xyh/rp3wBeWFUn0vUT9Hsjy7wAeG1Vbb8xz4nAb9F1H348cMqE9RwKXFdVv0jXB/8bRtb/3n79d01Y7nGSLANOAzb2kx4CXl5VzwVeBPynPojOAb5VVSdU1duSvJSuc7yTgROAk5K8cHfrk0bZ+6ierF4MrBnpGfKIvlfKI4GLk6ym++n/gSPLfK6qRvuc/1pVbQVIchNdXzNfHFvPw8Cn++EbgJf0wy8AfqUf/hhd19WT/MTIc99Ad+Mi6Lq4+L3+Q/3HdEcKPzVh+Zf2jxv78cPogmGXvaFKowwCPVkdQNez4w9HJyb5L8A1VfXy/nz7tSOzHxx7jr8dGf4Rk98vj9SOC23ztdmVH1bVCUmOpAuUNwHvA36NruO9k6rqkSTfpuubZlyA36+qDz7B9UqP8dSQnqw+C5y9fSTJCf3gkXQ9mcI+7Dd/guvYcdeu9btrXFX3A28G/lWSA+nqvKcPgRfR3VQF4G/obgW53VXA6/r+8UlyTPp79UoLZRDoyeCQJFtHHm+h+1Cd7S+gbqK7ny90PXv+fpLt9x0eym8Bb0nyNbr7D9+/uwWq6ka6nizXA39MV/8c3dHBN/o29wJf6r9u+u6q+izdqaevJPk6XffLh09cgTQPvz4qDSDJIXSnfSrJemBDVa3b3XLSYvAagTSMk4D399/0+T7wukWuR5qXRwSS1DivEUhS4wwCSWqcQSBJjTMIJKlxBoEkNe7/A6i/O5cFrNsLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#learner.data.batch_size = 512\n",
    "#learner.opt_func = RAdam\n",
    "learner.lr_find()\n",
    "learner.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ShowGraph(LearnerCallback):\n",
    "    \"Update a graph of learner stats and metrics after each epoch.\"\n",
    "    def on_epoch_end(self, n_epochs:int, last_metrics:MetricsList, **kwargs)->bool:\n",
    "        \"If we have `last_metrics` plot them in our pbar graph\"\n",
    "        if last_metrics is not None and last_metrics[0] is not None:\n",
    "            rec = self.learn.recorder\n",
    "            iters = range_of(rec.losses)\n",
    "            val_iter = np.array(rec.nb_batches).cumsum()\n",
    "            x_bounds = (0, (n_epochs - len(rec.nb_batches)) * rec.nb_batches[-1] + len(rec.losses))\n",
    "            y_bounds = (min((min(Tensor(rec.losses)), min(Tensor(rec.val_losses)))), \n",
    "                        max((max(Tensor(rec.losses)), max(Tensor(rec.val_losses)))))\n",
    "            rec.pbar.update_graph([(iters, rec.losses), (val_iter, rec.val_losses)], x_bounds, y_bounds)\n",
    "        return {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-19T17:12:05.721975Z",
     "start_time": "2019-08-19T17:12:05.714116Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False),\n",
       " __main__.ShowGraph]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.callback_fns.append(ShowGraph)\n",
    "learner.callback_fns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:21:04.766711Z",
     "start_time": "2019-08-19T17:12:17.245500Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='14' class='' max='20', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      70.00% [14/20 2:15:52<58:13]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>👉🏻LMAE👈🏻</th>\n",
       "      <th>lmae0</th>\n",
       "      <th>lmae1</th>\n",
       "      <th>lmae2</th>\n",
       "      <th>lmae3</th>\n",
       "      <th>lmae4</th>\n",
       "      <th>lmae5</th>\n",
       "      <th>lmae6</th>\n",
       "      <th>lmae7</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-0.503652</td>\n",
       "      <td>-0.285680</td>\n",
       "      <td>-3.061566</td>\n",
       "      <td>-2.026410</td>\n",
       "      <td>-3.519640</td>\n",
       "      <td>-2.193964</td>\n",
       "      <td>-3.489007</td>\n",
       "      <td>-3.072478</td>\n",
       "      <td>-3.589993</td>\n",
       "      <td>-2.933783</td>\n",
       "      <td>-3.667252</td>\n",
       "      <td>09:38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-0.515241</td>\n",
       "      <td>-0.285418</td>\n",
       "      <td>-3.060714</td>\n",
       "      <td>-2.028266</td>\n",
       "      <td>-3.520264</td>\n",
       "      <td>-2.191895</td>\n",
       "      <td>-3.486295</td>\n",
       "      <td>-3.072368</td>\n",
       "      <td>-3.588154</td>\n",
       "      <td>-2.933845</td>\n",
       "      <td>-3.664623</td>\n",
       "      <td>09:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-0.520008</td>\n",
       "      <td>-0.284994</td>\n",
       "      <td>-3.056636</td>\n",
       "      <td>-2.027188</td>\n",
       "      <td>-3.519684</td>\n",
       "      <td>-2.188406</td>\n",
       "      <td>-3.478980</td>\n",
       "      <td>-3.070544</td>\n",
       "      <td>-3.578715</td>\n",
       "      <td>-2.931054</td>\n",
       "      <td>-3.658517</td>\n",
       "      <td>09:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-0.522750</td>\n",
       "      <td>-0.285171</td>\n",
       "      <td>-3.060639</td>\n",
       "      <td>-2.022012</td>\n",
       "      <td>-3.525359</td>\n",
       "      <td>-2.189735</td>\n",
       "      <td>-3.485104</td>\n",
       "      <td>-3.071544</td>\n",
       "      <td>-3.586016</td>\n",
       "      <td>-2.935460</td>\n",
       "      <td>-3.669883</td>\n",
       "      <td>09:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-0.528367</td>\n",
       "      <td>-0.285469</td>\n",
       "      <td>-3.058288</td>\n",
       "      <td>-2.022976</td>\n",
       "      <td>-3.520865</td>\n",
       "      <td>-2.191772</td>\n",
       "      <td>-3.479712</td>\n",
       "      <td>-3.069207</td>\n",
       "      <td>-3.584090</td>\n",
       "      <td>-2.933733</td>\n",
       "      <td>-3.663950</td>\n",
       "      <td>09:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>-0.529909</td>\n",
       "      <td>-0.285104</td>\n",
       "      <td>-3.054100</td>\n",
       "      <td>-2.022380</td>\n",
       "      <td>-3.516894</td>\n",
       "      <td>-2.188716</td>\n",
       "      <td>-3.471087</td>\n",
       "      <td>-3.067172</td>\n",
       "      <td>-3.575789</td>\n",
       "      <td>-2.932689</td>\n",
       "      <td>-3.658071</td>\n",
       "      <td>09:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>-0.535584</td>\n",
       "      <td>-0.285300</td>\n",
       "      <td>-3.054309</td>\n",
       "      <td>-2.021156</td>\n",
       "      <td>-3.515384</td>\n",
       "      <td>-2.190034</td>\n",
       "      <td>-3.470951</td>\n",
       "      <td>-3.069556</td>\n",
       "      <td>-3.576298</td>\n",
       "      <td>-2.933831</td>\n",
       "      <td>-3.657264</td>\n",
       "      <td>09:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>-0.546055</td>\n",
       "      <td>-0.285952</td>\n",
       "      <td>-3.054501</td>\n",
       "      <td>-2.016385</td>\n",
       "      <td>-3.520241</td>\n",
       "      <td>-2.191372</td>\n",
       "      <td>-3.470513</td>\n",
       "      <td>-3.068488</td>\n",
       "      <td>-3.575293</td>\n",
       "      <td>-2.931984</td>\n",
       "      <td>-3.661729</td>\n",
       "      <td>09:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>-0.551168</td>\n",
       "      <td>-0.285836</td>\n",
       "      <td>-3.053061</td>\n",
       "      <td>-2.015641</td>\n",
       "      <td>-3.516887</td>\n",
       "      <td>-2.190797</td>\n",
       "      <td>-3.470135</td>\n",
       "      <td>-3.067873</td>\n",
       "      <td>-3.574162</td>\n",
       "      <td>-2.931169</td>\n",
       "      <td>-3.657826</td>\n",
       "      <td>09:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>-0.557653</td>\n",
       "      <td>-0.285969</td>\n",
       "      <td>-3.053200</td>\n",
       "      <td>-2.014346</td>\n",
       "      <td>-3.515790</td>\n",
       "      <td>-2.194010</td>\n",
       "      <td>-3.469678</td>\n",
       "      <td>-3.067449</td>\n",
       "      <td>-3.573705</td>\n",
       "      <td>-2.930238</td>\n",
       "      <td>-3.660380</td>\n",
       "      <td>09:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>-0.563965</td>\n",
       "      <td>-0.286118</td>\n",
       "      <td>-3.050467</td>\n",
       "      <td>-2.016622</td>\n",
       "      <td>-3.512140</td>\n",
       "      <td>-2.193799</td>\n",
       "      <td>-3.461238</td>\n",
       "      <td>-3.065708</td>\n",
       "      <td>-3.568330</td>\n",
       "      <td>-2.929677</td>\n",
       "      <td>-3.656223</td>\n",
       "      <td>09:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>-0.567964</td>\n",
       "      <td>-0.285992</td>\n",
       "      <td>-3.049241</td>\n",
       "      <td>-2.014698</td>\n",
       "      <td>-3.513150</td>\n",
       "      <td>-2.193004</td>\n",
       "      <td>-3.461623</td>\n",
       "      <td>-3.064822</td>\n",
       "      <td>-3.566189</td>\n",
       "      <td>-2.929189</td>\n",
       "      <td>-3.651256</td>\n",
       "      <td>09:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>-0.572177</td>\n",
       "      <td>-0.285987</td>\n",
       "      <td>-3.048324</td>\n",
       "      <td>-2.014721</td>\n",
       "      <td>-3.510280</td>\n",
       "      <td>-2.193889</td>\n",
       "      <td>-3.460192</td>\n",
       "      <td>-3.065171</td>\n",
       "      <td>-3.564825</td>\n",
       "      <td>-2.928029</td>\n",
       "      <td>-3.649487</td>\n",
       "      <td>09:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>-0.578990</td>\n",
       "      <td>-0.285861</td>\n",
       "      <td>-3.048306</td>\n",
       "      <td>-2.016231</td>\n",
       "      <td>-3.511709</td>\n",
       "      <td>-2.192702</td>\n",
       "      <td>-3.460806</td>\n",
       "      <td>-3.065513</td>\n",
       "      <td>-3.564143</td>\n",
       "      <td>-2.927918</td>\n",
       "      <td>-3.647420</td>\n",
       "      <td>09:40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='691', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with 👉🏻LMAE👈🏻 value: -3.06156587600708.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU9b3/8dcnCwQIW1gDAYIKyiIEiKwuVMGyKNAr16LUUpcfLv3VqtdqKNeq1w3vba21ti61KrYutWjFFlwAWbyK2lARI4tBQAkgBBQMCAGS7/1jToZJMpNthiTDeT8fj3HO+X6/58znzJH55JzvOd9jzjlERMS/Eho6ABERaVhKBCIiPqdEICLic0oEIiI+p0QgIuJzSQ0dQF20b9/eZWZm1m3hI4fgq43gIPAf76qp0PngtDQ8A/NeJHjT3jsh0xb6N40L2X0V9mOlq+Qi7WerMGuV6y1Mu7L5SHVWoV3Y8pCYwv6/WIv6aET6Dmv6/SYkQNrJsYlFYmLVqlW7nXMdKpbHZSLIzMwkNze3bgvvK4Dl94MlQkJi4N0SvOmEY2Xl6hLKl5W1TUyGhCRISIZE7z0hKaS8bLqsvkJbs8A/HFca8qo4X/EVWl8SeC89CqVl0yWBeVcSUlZWX1K5rSvxtikpsF1lcYedj9DGEgj8OJRCaVWxl4TfvrKYjxZDSTEcPey9F5cvO3oISg57Zd58cLo4EAMREkWlaQtfXrYdZXFChfmK9aHzoWUh21lxGefClHntSkuqjt0SCCaO4HRoeei2RKm67yvs54ck5aYt4ZLno49DYsbMPg9XHpeJICqtM2DSbxs6ChGRRkN9BCIiPqdEICLic/47NSQivnTkyBEKCgo4dOhQQ4dy3KWkpJCRkUFycnKN2isRiIgvFBQU0LJlSzIzM7FKV4GdOJxz7Nmzh4KCAnr27FmjZXRqSER84dChQ7Rr1+6ETgIAZka7du1qdeQTdSIwszQzW2Rm+d572zBtepjZKjNbbWafmNk1IXVDzOxjM9toZg/Zib6XRKTB+OXnpbbbGYsjghxgiXOuF7DEm69oBzDSOZcFDANyzKyLV/cIMBPo5b3GxSAmERGpoVgkgsnAXG96LjClYgPn3GHnXLE327Tsc80sHWjlnFvpAg9GeCbc8iIi8W7v3r38/ve/r/VyEyZMYO/evcchomNikQg6Oed2AHjvHcM1MrNuZrYG2Arc75zbDnQFCkKaFXhl4ZafaWa5ZpZbWFgYg7BFROpPpERQUlJS5XILFy6kTZs2xyssoIZXDZnZYqBzmKrZNf0g59xWYIB3SugVM5tH+Pvgww6U4px7HHgcIDs7WwMBiUhcycnJ4bPPPiMrK4vk5GRSU1NJT09n9erVrF27lilTprB161YOHTrET3/6U2bOnAkcG1Jn//79jB8/njPPPJN3332Xrl27Mn/+fJo1axZ1bDVKBM65MZHqzGynmaU753Z4p3p2VbOu7Wb2CXAW8A6QEVKdAWyvSUwiInV1598/Ye32b2K6zr5dWnH7hf0i1s+ZM4e8vDxWr17NsmXLmDhxInl5ecFLPJ988knS0tI4ePAgZ5xxBhdddBHt2rUrt478/Hyef/55/vCHP3DxxRfz0ksv8YMf/CDq2GNxauhVYIY3PQOYX7GBmWWYWTNvui0wCtjgnUoqMrPh3tVCPwy3vIjIiWbo0KHlrvN/6KGHGDhwIMOHD2fr1q3k5+dXWqZnz55kZWUBMGTIELZs2RKTWGJxQ9kc4EUzuxL4Avh3ADPLBq5xzl0F9AF+ZWbe8JD80jn3sbf8tcDTQDPgNe8lInLcVPWXe31p0aJFcHrZsmUsXryYlStX0rx5c0aPHh32PoCmTZsGpxMTEzl48GBMYok6ETjn9gDnhSnPBa7yphcBAyIsnwv0jzYOEZHGrGXLlhQVFYWt27dvH23btqV58+asX7+e9957r15j0xATIiL1oF27dowaNYr+/fvTrFkzOnXqFKwbN24cjz76KAMGDODUU09l+PDh9RqbuUpPbGr8srOzXZ0fTCMivrRu3Tr69OnT0GHUm3Dba2arnHPZFdtqrCEREZ9TIhAR8TklAhERn1MiEBHxOSUCERGfUyIQEfE5JQIRkUYoNTUVgO3btzN16tSwbUaPHk0sLqVXIhARacS6dOnCvHnzjutn6M5iEZF6cOutt9KjRw+uu+46AO644w7MjBUrVvD1119z5MgR7r77biZPnlxuuS1btnDBBReQl5fHwYMHufzyy1m7di19+vRpPGMNiYjEnddy4MuPq29XG51Ph/FzIlZPmzaNG264IZgIXnzxRV5//XVuvPFGWrVqxe7duxk+fDiTJk2K+MzhRx55hObNm7NmzRrWrFnD4MGDYxK6EoGISD0YNGgQu3btYvv27RQWFtK2bVvS09O58cYbWbFiBQkJCWzbto2dO3fSuXO454DBihUruP766wEYMGAAAwaEHcuz1pQIRMR/qvjL/XiaOnUq8+bN48svv2TatGk8++yzFBYWsmrVKpKTk8nMzAw7/HSoSEcL0VBnsYhIPZk2bRovvPAC8+bNY+rUqezbt4+OHTuSnJzM0qVL+fzzz6tc/uyzz+bZZ58FIC8vjzVr1sQkLh0RiIjUk379+lFUVETXrl1JT09n+vTpXHjhhWRnZ5OVlcVpp51W5fLXXnstl19+OQMGDCArK4uhQ4fGJC4NQy0ivqBhqDUMtYiIRKBEICLic0oEIuIb8XgqvC5qu51KBCLiCykpKezZs+eETwbOOfbs2UNKSkqNl9FVQyLiCxkZGRQUFFBYWNjQoRx3KSkpZGRk1Li9EoGI+EJycjI9e/Zs6DAaJZ0aEhHxOSUCERGfUyIQEfE5JQIREZ+LKhGYWZqZLTKzfO+9bZg2PcxslZmtNrNPzOyakLplZrbBq1ttZh2jiUdERGov2iOCHGCJc64XsMSbr2gHMNI5lwUMA3LMrEtI/XTnXJb32hVlPCIiUkvRJoLJwFxvei4wpWID59xh51yxN9s0Bp8pIiIxFO2Pcifn3A4A7z3sqR0z62Zma4CtwP3Oue0h1U95p4VusyqeuGBmM80s18xy/XBDiIhIfak2EZjZYjPLC/OaXN2yZZxzW51zA4BTgBlm1smrmu6cOx04y3tdVsU6HnfOZTvnsjt06FDTjxYRkWpUe2exc25MpDoz22lm6c65HWaWDlR5jt85t93MPiHwoz/PObfNKy8ys+eAocAztdoCERGJSrSnhl4FZnjTM4D5FRuYWYaZNfOm2wKjgA1mlmRm7b3yZOACIC/KeEREpJaiTQRzgLFmlg+M9eYxs2wze8Jr0wd438w+ApYDv3TOfUyg4/gNr+9gNbAN+EOU8YiISC3pUZUiIj6hR1WKiEhYSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNRJwIzSzOzRWaW7723raJtKzPbZmYPh5QNMbOPzWyjmT1kZhZtTCIiUnOxOCLIAZY453oBS7z5SO4CllcoewSYCfTyXuNiEJOIiNRQLBLBZGCuNz0XmBKukZkNAToBb4aUpQOtnHMrnXMOeCbS8iIicnzEIhF0cs7tAPDeO1ZsYGYJwK+An1Wo6goUhMwXeGUiIlJPkmrSyMwWA53DVM2u4edcByx0zm2t0AUQrj/ARYhhJoFTSHTv3r2GHysiItWpUSJwzo2JVGdmO80s3Tm3wzvVsytMsxHAWWZ2HZAKNDGz/cBvgIyQdhnA9ggxPA48DpCdnR02WYiISO3F4tTQq8AMb3oGML9iA+fcdOdcd+dcJnAz8IxzLsc7lVRkZsO9q4V+GG55ERE5fmKRCOYAY80sHxjrzWNm2Wb2RA2WvxZ4AtgIfAa8FoOYRESkhixwsU58yc7Odrm5uQ0dhohIXDGzVc657IrlurNYRMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfC6qRGBmaWa2yMzyvfe2VbRtZWbbzOzhkLJlZrbBzFZ7r47RxCMiIrUX7RFBDrDEOdcLWOLNR3IXsDxM+XTnXJb32lWTD3Wu9oGKiEh40SaCycBcb3ouMCVcIzMbAnQC3ozy8wD4rHB/LFYjIiJEnwg6Oed2AHjvlU7tmFkC8CvgZxHW8ZR3Wug2M7NIH2RmM80s18xyDx4piTJsEREpk1RdAzNbDHQOUzW7hp9xHbDQObc1zO/8dOfcNjNrCbwEXAY8E24lzrnHgccBmqb30skhEZEYqTYROOfGRKozs51mlu6c22Fm6UC4c/wjgLPM7DogFWhiZvudcznOuW3eZxSZ2XPAUCIkgjBxUcUBhIiI1FC0p4ZeBWZ40zOA+RUbOOemO+e6O+cygZuBZ5xzOWaWZGbtAcwsGbgAyKvpB//ri71Rhi4iIhB9IpgDjDWzfGCsN4+ZZZvZE9Us2xR4w8zWAKuBbcAfavrBFz3ybt0iFhGRcqo9NVQV59we4Lww5bnAVWHKnwae9qYPAEOi+fxVn3/FkB5p0axCRMT34vrO4oseWdnQIYiIxL24TgQiIhK9uEwEfdNbMWNED1KbRnVmS0REiNNEkJhgrN3xDfuLj/LpzqKGDkdEJK7FZSIAuHRYdwD+9fnXDRyJiEh8i9tEMOrk9gCs3qr7CUREohG3iaBdalMAXvjn1gaOREQkvsVtIkhMODa8xCENQiciUmdxmwhC3fTi6oYOQUQkbsV1Ilhw/ZkAnNQ+tYEjERGJX3GdCPqmtyK1aRL7i482dCgiInErrhOBmZHRthkFXx9s6FBEROJWXCcCwEsE3zZ0GCIicSvuE8FnhQdY/2URW3YfaOhQRETiUtwngoEZrQFYpTuMRUTqJO4TweyJfQH4j79+1MCRiIjEp7hPBO1TmwSnvz2sq4dERGor7hOBmdEnvRUAL/1rWwNHIyISf+I+EQDM+bfTAfhIA9CJiNTaCZEITu8a6DCet6qAbXt1T4GISG2cEIkgIWQAuvsWrmvASERE4s8JkQgAlt48GoCWKXp8pYhIbZwwiaBn+xac1KEFf/tQHcYiIrVxwiQCgNSmSRw6Usp9r+n0kIhITZ1QiWDGiEwAHlu+iQVrdjRsMCIiceKESgT/NrhrcPrHz/2rASMREYkfJ1QiMDP+99bvBOd3FR1qwGhEROJDVInAzNLMbJGZ5XvvbSO0KzGz1d7r1ZDynmb2vrf8X8ysSbjlayOjbXN+9t1TAXjhAz3YXkSkOtEeEeQAS5xzvYAl3nw4B51zWd5rUkj5/cCvveW/Bq6MMh4Arj3nZAAeWPQpzrlYrFJE5IQVbSKYDMz1pucCU2q6oJkZcC4wry7LVyX0BrOesxZSUuo4eLgkFqsWETnhRJsIOjnndgB47x0jtEsxs1wze8/Myn7s2wF7nXNlQ4YWAF3DLw5mNtNbR25hYWG1gT1zxdDg9Mk/X0ifX7xOaamODkREKqo2EZjZYjPLC/OaXIvP6e6cywYuBR40s5MBC9Mu4i+1c+5x51y2cy67Q4cO1X7g2b07MLRnWrmyAXe+ydGSUs771TLWFGiAOhERqEEicM6Ncc71D/OaD+w0s3QA731XhHVs9943AcuAQcBuoI2ZlY0JkQFsj3qLQvz5ymHl5vcXH2Xsr1fwWeEBJj38Du9u3B3LjxMRiUvRnhp6FZjhTc8A5ldsYGZtzaypN90eGAWsdYFe3KXA1KqWj0aTpAQ23jOejfeMD5ZtDnm28aVPvA/AkZJSMnMWkJmzIJYfLyISF6JNBHOAsWaWD4z15jGzbDN7wmvTB8g1s48I/PDPcc6t9epuBW4ys40E+gz+GGU8lSQlJpCUmMD0Yd3D1mfmLKDX7Ndi/bEiInHD4vHyyuzsbJebm1urZZxzPPnOFv7+0XbunNSPyb97J2y7Fk0S+eS/xsUiTBGRRsXMVnn9teWcUHcWV8XMuPLMnrzy41EM7NaGq885KWy7A4dLOFB8lIOHS8jMWcDI+5bUc6QiIvXLN4mgolnj+7Dp3gm8m3Nupbp+t7/B3JVbANi+LzBMxfzV27j40ZW6QU1ETji+OTVUla8OHKZNs2QOl5Ry2m2vV9n2kqHduPd7pxO4H05EJH74/tRQVdJaNCEhwUhJTqy27fMfbKXnrIU88fYmfrM4vx6iExE5vpQIKvhg9nn81+R+1ba7e8E6fr34Uz7dWRQ8XfRxwT5yt3x1vEMUEYkpPeC3go4tU/jhiEwOHi6hZ/sWnN+vM0vW7WT3/mK+f0Z3Tr/9DYqKjwbbn//rFQAkGJSNYHHP9/qzu+gwPx3Tq9L6S0sddy1YyyVDu9O7U8t62SYRkaqoj6AObv7rR+wqKmbFp1WPeTR1SAZdWqdQsPcgd0zqx/gH3+a2C/pyzZ9XAbDg+jPp16V1fYQsIhKxj0CJIAr/9fe1PPnO5qjWsWXOxBhFIyJSNXUWHwe/uLAvW+ZMZOH1Z9V5HTOfySUzZwGv531Zqe7NT74sN2Lq0ZJSBt+1iMycBfzyjQ11/kwRkVA6IoiRXUWHaNOsCZ9s38flT/+TBy4eyBVP59I+tSm79xfXaB2f3TuBRO9ZCrfPz2Puys8B+O+LBrDnwGHuf319ufav/HgUPdu1oHXz5NhujIickHRqqAEsWruT7B5tWbfjGw4cLmFs305cNTeXts2Tuf+iAZz084Xl2k/O6sJvpg3iaEkpU37/DnnbvqnR50Q6vbR2+ze0bp5M1zbNgmU7vznExwX7MIPz+nSq+8aJSNyJlAh01dBxNLZv4Id25Cntg2VPzKi0D4Lmr97Ory/O4uUPt1WZBF6/4SzGPfh2xPrL/vg+b+cHhthOMNh037FEcc2fV/HhF8eexaA+ChFRH0ED+tHITADWhQxy9/7mr7hl3prg/LQzugWnbxl3Ku/NOo/TOrciu0fbYPm+g0fKrbcsCcCxS1qXrt9FZs6CckkA0JAZIqJE0JDumNSPLXMm0qxJIndOCtzEdskf3gvWr79rHHMuGsB1o0/mnN4duG70KXRunQLAvGtHcveU/gD8NXdrlZ+zqXA/lz/9z7B1p9/xJt8cOhK2TkT8QYmgkfjhiB7l5ts0Tw4OeXHLuNOYG/IM5jKDurcBAnc5l/1lXxLmuczn/mp5uflff38gL149Agg8tW3Gkx/UOt6xDywPHmXogT4i8U2JoJEwM2aEJIPVvzi/2mVCb0brOWshz6zcwsleB/T15/UiP+TJbAB3TurHx3ecz/cGZdCjXfNgeUbb5tTGw2/lk7+r/FFGaZgEJCLxQZ3Fjcidk/tz2YgenNKxbkNP/GL+J8HpXh1TSU4sn+cvG96DBO/y1E6tUoLlf/9oO7+9ZBAABV9/y5n3L2XigHR+d+lgDhQf5aEl+Qw/qR0v5m7ltTD3OwAU7i8ut04RiR86ImhkapsE/hjhKqQhXmfy4pvOAaB5k8RgEigTesXQ3He3APDY8k0ALFizg+KjJfS7/Q0eW7GJy5/+Z8QkAPBZ4f5axS0ijYcSQZw7r08nHpk+uFzZpnsn0MW7d+CUjqlsvm8CeXd8N+zyN3gD493+auBo4mhpabDu1P+M/GyG/3dWTwAGdgv0U2zcVfdEsKvoELu+OVTn5UUkOkoEJ4Dxp6czY0QPXpg5nC1zJlb6y9/MKpWV+cm5x0ZInflMLs9/UPUVSADd0poxe2JfNt83gVeuG0nzJonk79zPAW9U1tJSx2V/fJ+Cr7+ttGzFzuzSUsfQe5Yw9F49ElSkoejOYuH7j63k/c3ln6NwwYB0/rFmBwAf3X4+A+98M+JoqaFXDc08+yT+mruVr789dknqS9eOYHD3tvScFejILjsllbvlK6Y+ujLYbv1d42r0cCARqRvdWSwRTRyQXi4RpCQn8PClg3loWuCPhIQEq/EdyI+v2FSp7KJHVnJ612MJ5LHln3HRkIxySQCgsKiYbmm1u4JJRKKnU0PCZcN7sPTm0cH5edeMBAIJINIppVChdz9H8vG2fcHp+15bT/bdiyu1+bKafoLSUseuIvUliMSaEoFgZvRs34KF15/FJ3d+l/5da/ewnLum9OeOC/uWK/vzlcPYcPe4CEscc9PY3rx549kA7NhX9Y/8b9/ayNB7ljBqzlu1ik9EqqZTQxLUt0urOi2XnJjAj0b1pMTBXf9YC8CZvQID7W2ZM5Gb/rKalz/cxgszh7N+xzfc8fe1wWV/cu4pwUd/Xv/8h0wa2AWA/J1F5Lz8Mas+/zrYtmPLpgBs23sQgI+27mXR2p2kpiTRokkil43IrFP8In6nRCAxc8WoTF75cBs3je1drvyB72fxwPezAMjq1oaH3tqIAW/f+h3MjJZNj/1v+MWeb0lNSWKs9yzoULuKjj3X4SfPf8jfP9pern7YSe30HGiROtBVQ9IoxGK8ou+c2oGnLq88JpOIBByXR1WaWZqZLTKzfO+9bYR2JWa22nu9GlL+tJltDqnLiiYeiV+r/nNM2PIbx/Tm8cuGML5/ZwCuPvukiOtYuqEwOPheaamr1fhH+w4eCd4HIeI30Z4aygGWOOfmmFmON39rmHYHnXORfuR/5pybF2UcEufapTatVBZ6yer5/ToHp8efns6U373Dby8ZxIVen0LZEcW9C9cxe2Lf4NPf3s05N3iXdSRHS0oZeOeblT5TxC+ivWpoMjDXm54LTIlyfeJj15xzcnD6uauGRWyX1a0Nm+6dEEwCAJcO6w4EnvJWFPJ8hZFz3goeJfSavZAJvzn2ZLdDR0rIzFnAKbNfC5YdKTk2xIaIX0TVR2Bme51zbULmv3bOVTo9ZGZHgdXAUWCOc+4Vr/xpYARQDCwBcpxzYZ/0bmYzgZkA3bt3H/L555/XOW5pnEpKHb9bupExfTrV6QqmsqOClk2TglciQeCoYGSFS06vOedkHl3+WaV1/OMnZ9b68lmReFHnPgIzW2xmeWFek2vx+d29D78UeNDMyv70mwWcBpwBpBH+tBIAzrnHnXPZzrnsDh061OKjJV4kJhjXn9erzpexlilLAlefE+hP+OUbGyq1CZcEAA4eKQlbfuhICTf+ZTVrt0d+lrRIvKo2ETjnxjjn+od5zQd2mlk6gPe+K8I6tnvvm4BlwCBvfocLKAaeAnTJh8TMvw3KAODlD7dFbHPuaR0B6O4NbXHfwnVh25122+v87cNtTHjo7XLls15eQ2bOgnKno0TiTbR9BK8CM7zpGcD8ig3MrK2ZNfWm2wOjgLXefFkSMQL9C3lRxiM+9tK1I4PT7806j1M7t6xQP4ILBqQH51umJPHkj85g830T+Os1gUd3/uuLvZXWW3y08lHCOxt3k5mzIDha6+l3vKlHdkrciraPoB3wItAd+AL4d+fcV2aWDVzjnLvKzEYCjwGlBBLPg865P3rLvwV0AIxAH8I1zrlqB7bXfQQSyf7ioyQlWHAU05JSx9V/yqV3p5bcMu60YLt9B4/QullyuWVj8UP+H2N785PzelXfUKQBROoj0A1lIp7Ln/qApRsKo17Ppnsn1GiwPpH6dlxuKBM5kdx/0YAq628+v/zQGc9dNYx/3TaWj24/n1njjx1tvLdpT8R1LP+0kPydRdEFKhJjSgQino6tUvjzlcPoltaMa0efHOxPuP68XnRLa8aMkZk8+aPAH1MvXTuCkae0J61FE1o3S+bqc07mqR+dAcClT7wf9q7mvd8eZsaTH5QbR+lA8VEWrNnB4aO6f0Eajk4NicSIcy74FDaAMX06sXjdTp67ahif7iwqN+rqljkTy7Xv37UVXx84wi3jTmVyVtd6j138QX0EIvVgxH1Lqn2uAsArPx7FlN+9E7H+07vH0yRJB+wSW+ojEKkHb9/yHSac3rnadlUlAYBNu6u9eE4kZpQIRGIoKTGB314ymKGZaQzNTOPnE451Iq+cdS6/nz640jLLfza6UtmSdeXvzQw9cp/18hpe+XAbM578oFzH9K5vDrH800LWFFS+F0KkKjo1JFKP8ncWBTuLl908mrTUJrRKSea6Z1dxft/ODO2ZFhwXqWwk1LL7G5bePJpWKUkMqfC852vOOZnd+4uZt6ogWLbg+jPp10VjJkl56iMQaSR2FR2iQ2pTAjfUlxfagfzWf5xDakoSQ+9ZUqfP0ZDaUpH6CEQaiY4tU8ImAQAz4/9/5xQAPt/zbZ2TAMA3dRz/aNyDK8jMWcB+PajHN5QIRBqZy0dlAvDm2p0R24zv35kBGYFTP9ecczJn9+7A0Mw0tsyZSJfWKQB8vvvbaj/rtY93kJmzgI279lN06AiZOQtY/2Xghrftew9GuSUSL/TwepFGpuxpbc9/8EWwbPN9EzAzduw7SIfUpiQlBv6Gc85VOrq4fVI/rv7TKi58+H9Z/rPR7N5fTP+urfliz7ektWgSXH/o2ErTHn+P3fvLPwrk28Phh+SWE48SgUgjl3fnd4M/9umtyz92M9wppvapTYLT5/zPskr1W+ZM5Ln3vyhXVjEJAHx94HBdwpU4pEQg0silNq3dP9MhPdLo1TGV/F3h70Wo6Siredv20aZ5Mv26tK50c9szK7ewu6iYG8f2jtjfIfFDfQQijVDZ8xEmhTyXuTYW3XROjdp1bVP+CGPzfRO4YUxgGO1fLfqU7/3+XXr/57FnOjvn2LO/mF/M/4SH3trImoJ95Zbfs7+YL/ZU3zchjYuOCEQaoTO8jt9oPHbZEK7+0yo6tWpKj3Yt+GDzV+Xq1981jpTkREpKHcPuXczcK4ZiZtwwpjcPLs4v1/bw0VKaJCWUG0sJYMueAwzsFnxsebl7HCZndeHB72fpiCEO6D4CEZ84dKSE0257HYB/zh5Dh5ZNI7Ydes9idhUd6zcY3Im5xK4AAAYNSURBVL0Nz1w5jP63v1GpbfMmifx8Qh8GZrThwof/t1yd7mVoXHRDmYjUWKweu6lE0LjohjIRqbGlN4+usq6qH/hl3rIPXDwwxlHJ8aI+AhGppGf7Frx549l0T2vO5t0HGP+bt4N1me2aB9ts3n2g0rKZ7VvoSCDO6IhARMLq3aklKcmJ9O7UMlj20CWDgp2/z1wxlCZJCdw9pX+w/rWfnlXvcUr04rKPwMyKgA0NHUcU2gO7GzqIKMX7NsR7/BD/2xDv8UP8bUMP51yHioXxempoQ7gOj3hhZrnxHD/E/zbEe/wQ/9sQ7/HDibENoFNDIiK+p0QgIuJz8ZoIHm/oAKIU7/FD/G9DvMcP8b8N8R4/nBjbEJ+dxSIiEjvxekQgIiIxokQgIuJzcZUIzGycmW0ws41mltPQ8ZQxs25mttTM1pnZJ2b2U688zcwWmVm+997WKzcze8jbjjVmNjhkXTO89vlmNqMBtiXRzD40s3948z3N7H0vnr+YWROvvKk3v9GrzwxZxyyvfIOZfbceY29jZvPMbL23L0bE2z4wsxu9/4fyzOx5M0tp7PvAzJ40s11mlhdSFrPv3cyGmNnH3jIPWYyHM40Q//94/x+tMbO/mVmbkLqw322k36dI+69Rcc7FxQtIBD4DTgKaAB8BfRs6Li+2dGCwN90S+BToC/w3kOOV5wD3e9MTgNcAA4YD73vlacAm772tN922nrflJuA54B/e/IvANG/6UeBab/o64FFvehrwF2+6r7dvmgI9vX2WWE+xzwWu8qabAG3iaR8AXYHNQLOQ7/5HjX0fAGcDg4G8kLKYfe/AB8AIb5nXgPH1EP/5QJI3fX9I/GG/W6r4fYq0/xrTq8EDqMXOGgG8ETI/C5jV0HFFiHU+MJbA3c/pXlk6gRvhAB4DLglpv8GrvwR4LKS8XLt6iDsDWAKcC/zD+4e3O+QfRHAfAG8AI7zpJK+dVdwvoe2Oc+ytCPyIWoXyuNkHBBLBVu/HMMnbB9+Nh30AZFb4IY3J9+7VrQ8pL9fueMVfoe57wLPedNjvlgi/T1X9G2pMr3g6NVT2j6RMgVfWqHiH54OA94FOzrkdAN57R69ZpG1p6G18ELgFKPXm2wF7nXNHw8QTjNWr3+e1b6htOAkoBJ7yTm09YWYtiKN94JzbBvwS+ALYQeA7XUX87INQsfreu3rTFcvr0xUEjkSg9vFX9W+o0YinRBDuvGCjuvbVzFKBl4AbnHPfVNU0TJmrovy4M7MLgF3OuVWhxVXE09i2IYnA4f0jzrlBwAECpyQiaWzx451Hn0zglEMXoAUwvop4Gt021EBtY27QbTGz2cBR4NmyogjxNMr4ayqeEkEB0C1kPgPY3kCxVGJmyQSSwLPOuZe94p1mlu7VpwO7vPJI29KQ2zgKmGRmW4AXCJweehBoY2ZlY1KFxhOM1atvDXxFw21DAVDgnHvfm59HIDHE0z4YA2x2zhU6544ALwMjiZ99ECpW33uBN12x/LjzOqwvAKY777xONXGGK99N5P3XaMRTIvgn0MvrgW9CoHPs1QaOCQhcCQH8EVjnnHsgpOpVoOzqhxkE+g7Kyn/oXUExHNjnHT6/AZxvZm29vw7P98qOO+fcLOdchnMuk8B3+5ZzbjqwFJgaYRvKtm2q19555dO8K1p6Ar0IdPYd7/i/BLaa2ale0XnAWuJoHxA4JTTczJp7/0+VbUNc7IMKYvK9e3VFZjbc+05+GLKu48bMxgG3ApOcc99W2K5w323Y3ydvf0Taf41HQ3dS1OZF4IqDTwn0zs9u6HhC4jqTwOHeGmC195pA4PzgEiDfe0/z2hvwO287PgayQ9Z1BbDRe13eQNszmmNXDZ1E4H/0jcBfgaZeeYo3v9GrPylk+dnetm0gxld4VBN3FpDr7YdXCFx9Elf7ALgTWA/kAX8icHVKo94HwPME+jSOEPjL+MpYfu9Atvd9fAY8TIULAo5T/BsJnPMv+/f8aHXfLRF+nyLtv8b00hATIiI+F0+nhkRE5DhQIhAR8TklAhERn1MiEBHxOSUCERGfUyIQEfE5JQIREZ/7P6k60hy70mCHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-41-7fee45088556>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-6\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#, moms=(0.75,0.70))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[1;32m     20\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[1;32m     21\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[0;32m---> 22\u001b[0;31m     \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlr_find\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_it\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop_div\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mskip_bwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mskip_bwd\u001b[0m\u001b[0;34m:\u001b[0m                     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_step_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m     \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    116\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU9b3/8dcnCwQIW1gDAYIKyiIEiKwuVMGyKNAr16LUUpcfLv3VqtdqKNeq1w3vba21ti61KrYutWjFFlwAWbyK2lARI4tBQAkgBBQMCAGS7/1jToZJMpNthiTDeT8fj3HO+X6/58znzJH55JzvOd9jzjlERMS/Eho6ABERaVhKBCIiPqdEICLic0oEIiI+p0QgIuJzSQ0dQF20b9/eZWZm1m3hI4fgq43gIPAf76qp0PngtDQ8A/NeJHjT3jsh0xb6N40L2X0V9mOlq+Qi7WerMGuV6y1Mu7L5SHVWoV3Y8pCYwv6/WIv6aET6Dmv6/SYkQNrJsYlFYmLVqlW7nXMdKpbHZSLIzMwkNze3bgvvK4Dl94MlQkJi4N0SvOmEY2Xl6hLKl5W1TUyGhCRISIZE7z0hKaS8bLqsvkJbs8A/HFca8qo4X/EVWl8SeC89CqVl0yWBeVcSUlZWX1K5rSvxtikpsF1lcYedj9DGEgj8OJRCaVWxl4TfvrKYjxZDSTEcPey9F5cvO3oISg57Zd58cLo4EAMREkWlaQtfXrYdZXFChfmK9aHzoWUh21lxGefClHntSkuqjt0SCCaO4HRoeei2RKm67yvs54ck5aYt4ZLno49DYsbMPg9XHpeJICqtM2DSbxs6ChGRRkN9BCIiPqdEICLic/47NSQivnTkyBEKCgo4dOhQQ4dy3KWkpJCRkUFycnKN2isRiIgvFBQU0LJlSzIzM7FKV4GdOJxz7Nmzh4KCAnr27FmjZXRqSER84dChQ7Rr1+6ETgIAZka7du1qdeQTdSIwszQzW2Rm+d572zBtepjZKjNbbWafmNk1IXVDzOxjM9toZg/Zib6XRKTB+OXnpbbbGYsjghxgiXOuF7DEm69oBzDSOZcFDANyzKyLV/cIMBPo5b3GxSAmERGpoVgkgsnAXG96LjClYgPn3GHnXLE327Tsc80sHWjlnFvpAg9GeCbc8iIi8W7v3r38/ve/r/VyEyZMYO/evcchomNikQg6Oed2AHjvHcM1MrNuZrYG2Arc75zbDnQFCkKaFXhl4ZafaWa5ZpZbWFgYg7BFROpPpERQUlJS5XILFy6kTZs2xyssoIZXDZnZYqBzmKrZNf0g59xWYIB3SugVM5tH+Pvgww6U4px7HHgcIDs7WwMBiUhcycnJ4bPPPiMrK4vk5GRSU1NJT09n9erVrF27lilTprB161YOHTrET3/6U2bOnAkcG1Jn//79jB8/njPPPJN3332Xrl27Mn/+fJo1axZ1bDVKBM65MZHqzGynmaU753Z4p3p2VbOu7Wb2CXAW8A6QEVKdAWyvSUwiInV1598/Ye32b2K6zr5dWnH7hf0i1s+ZM4e8vDxWr17NsmXLmDhxInl5ecFLPJ988knS0tI4ePAgZ5xxBhdddBHt2rUrt478/Hyef/55/vCHP3DxxRfz0ksv8YMf/CDq2GNxauhVYIY3PQOYX7GBmWWYWTNvui0wCtjgnUoqMrPh3tVCPwy3vIjIiWbo0KHlrvN/6KGHGDhwIMOHD2fr1q3k5+dXWqZnz55kZWUBMGTIELZs2RKTWGJxQ9kc4EUzuxL4Avh3ADPLBq5xzl0F9AF+ZWbe8JD80jn3sbf8tcDTQDPgNe8lInLcVPWXe31p0aJFcHrZsmUsXryYlStX0rx5c0aPHh32PoCmTZsGpxMTEzl48GBMYok6ETjn9gDnhSnPBa7yphcBAyIsnwv0jzYOEZHGrGXLlhQVFYWt27dvH23btqV58+asX7+e9957r15j0xATIiL1oF27dowaNYr+/fvTrFkzOnXqFKwbN24cjz76KAMGDODUU09l+PDh9RqbuUpPbGr8srOzXZ0fTCMivrRu3Tr69OnT0GHUm3Dba2arnHPZFdtqrCEREZ9TIhAR8TklAhERn1MiEBHxOSUCERGfUyIQEfE5JQIRkUYoNTUVgO3btzN16tSwbUaPHk0sLqVXIhARacS6dOnCvHnzjutn6M5iEZF6cOutt9KjRw+uu+46AO644w7MjBUrVvD1119z5MgR7r77biZPnlxuuS1btnDBBReQl5fHwYMHufzyy1m7di19+vRpPGMNiYjEnddy4MuPq29XG51Ph/FzIlZPmzaNG264IZgIXnzxRV5//XVuvPFGWrVqxe7duxk+fDiTJk2K+MzhRx55hObNm7NmzRrWrFnD4MGDYxK6EoGISD0YNGgQu3btYvv27RQWFtK2bVvS09O58cYbWbFiBQkJCWzbto2dO3fSuXO454DBihUruP766wEYMGAAAwaEHcuz1pQIRMR/qvjL/XiaOnUq8+bN48svv2TatGk8++yzFBYWsmrVKpKTk8nMzAw7/HSoSEcL0VBnsYhIPZk2bRovvPAC8+bNY+rUqezbt4+OHTuSnJzM0qVL+fzzz6tc/uyzz+bZZ58FIC8vjzVr1sQkLh0RiIjUk379+lFUVETXrl1JT09n+vTpXHjhhWRnZ5OVlcVpp51W5fLXXnstl19+OQMGDCArK4uhQ4fGJC4NQy0ivqBhqDUMtYiIRKBEICLic0oEIuIb8XgqvC5qu51KBCLiCykpKezZs+eETwbOOfbs2UNKSkqNl9FVQyLiCxkZGRQUFFBYWNjQoRx3KSkpZGRk1Li9EoGI+EJycjI9e/Zs6DAaJZ0aEhHxOSUCERGfUyIQEfE5JQIREZ+LKhGYWZqZLTKzfO+9bZg2PcxslZmtNrNPzOyakLplZrbBq1ttZh2jiUdERGov2iOCHGCJc64XsMSbr2gHMNI5lwUMA3LMrEtI/XTnXJb32hVlPCIiUkvRJoLJwFxvei4wpWID59xh51yxN9s0Bp8pIiIxFO2Pcifn3A4A7z3sqR0z62Zma4CtwP3Oue0h1U95p4VusyqeuGBmM80s18xy/XBDiIhIfak2EZjZYjPLC/OaXN2yZZxzW51zA4BTgBlm1smrmu6cOx04y3tdVsU6HnfOZTvnsjt06FDTjxYRkWpUe2exc25MpDoz22lm6c65HWaWDlR5jt85t93MPiHwoz/PObfNKy8ys+eAocAztdoCERGJSrSnhl4FZnjTM4D5FRuYWYaZNfOm2wKjgA1mlmRm7b3yZOACIC/KeEREpJaiTQRzgLFmlg+M9eYxs2wze8Jr0wd438w+ApYDv3TOfUyg4/gNr+9gNbAN+EOU8YiISC3pUZUiIj6hR1WKiEhYSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNKBCIiPqdEICLic0oEIiI+p0QgIuJzSgQiIj6nRCAi4nNRJwIzSzOzRWaW7723raJtKzPbZmYPh5QNMbOPzWyjmT1kZhZtTCIiUnOxOCLIAZY453oBS7z5SO4CllcoewSYCfTyXuNiEJOIiNRQLBLBZGCuNz0XmBKukZkNAToBb4aUpQOtnHMrnXMOeCbS8iIicnzEIhF0cs7tAPDeO1ZsYGYJwK+An1Wo6goUhMwXeGUiIlJPkmrSyMwWA53DVM2u4edcByx0zm2t0AUQrj/ARYhhJoFTSHTv3r2GHysiItWpUSJwzo2JVGdmO80s3Tm3wzvVsytMsxHAWWZ2HZAKNDGz/cBvgIyQdhnA9ggxPA48DpCdnR02WYiISO3F4tTQq8AMb3oGML9iA+fcdOdcd+dcJnAz8IxzLsc7lVRkZsO9q4V+GG55ERE5fmKRCOYAY80sHxjrzWNm2Wb2RA2WvxZ4AtgIfAa8FoOYRESkhixwsU58yc7Odrm5uQ0dhohIXDGzVc657IrlurNYRMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfE6JQETE55QIRER8TolARMTnlAhERHxOiUBExOeUCEREfC6qRGBmaWa2yMzyvfe2VbRtZWbbzOzhkLJlZrbBzFZ7r47RxCMiIrUX7RFBDrDEOdcLWOLNR3IXsDxM+XTnXJb32lWTD3Wu9oGKiEh40SaCycBcb3ouMCVcIzMbAnQC3ozy8wD4rHB/LFYjIiJEnwg6Oed2AHjvlU7tmFkC8CvgZxHW8ZR3Wug2M7NIH2RmM80s18xyDx4piTJsEREpk1RdAzNbDHQOUzW7hp9xHbDQObc1zO/8dOfcNjNrCbwEXAY8E24lzrnHgccBmqb30skhEZEYqTYROOfGRKozs51mlu6c22Fm6UC4c/wjgLPM7DogFWhiZvudcznOuW3eZxSZ2XPAUCIkgjBxUcUBhIiI1FC0p4ZeBWZ40zOA+RUbOOemO+e6O+cygZuBZ5xzOWaWZGbtAcwsGbgAyKvpB//ri71Rhi4iIhB9IpgDjDWzfGCsN4+ZZZvZE9Us2xR4w8zWAKuBbcAfavrBFz3ybt0iFhGRcqo9NVQV59we4Lww5bnAVWHKnwae9qYPAEOi+fxVn3/FkB5p0axCRMT34vrO4oseWdnQIYiIxL24TgQiIhK9uEwEfdNbMWNED1KbRnVmS0REiNNEkJhgrN3xDfuLj/LpzqKGDkdEJK7FZSIAuHRYdwD+9fnXDRyJiEh8i9tEMOrk9gCs3qr7CUREohG3iaBdalMAXvjn1gaOREQkvsVtIkhMODa8xCENQiciUmdxmwhC3fTi6oYOQUQkbsV1Ilhw/ZkAnNQ+tYEjERGJX3GdCPqmtyK1aRL7i482dCgiInErrhOBmZHRthkFXx9s6FBEROJWXCcCwEsE3zZ0GCIicSvuE8FnhQdY/2URW3YfaOhQRETiUtwngoEZrQFYpTuMRUTqJO4TweyJfQH4j79+1MCRiIjEp7hPBO1TmwSnvz2sq4dERGor7hOBmdEnvRUAL/1rWwNHIyISf+I+EQDM+bfTAfhIA9CJiNTaCZEITu8a6DCet6qAbXt1T4GISG2cEIkgIWQAuvsWrmvASERE4s8JkQgAlt48GoCWKXp8pYhIbZwwiaBn+xac1KEFf/tQHcYiIrVxwiQCgNSmSRw6Usp9r+n0kIhITZ1QiWDGiEwAHlu+iQVrdjRsMCIiceKESgT/NrhrcPrHz/2rASMREYkfJ1QiMDP+99bvBOd3FR1qwGhEROJDVInAzNLMbJGZ5XvvbSO0KzGz1d7r1ZDynmb2vrf8X8ysSbjlayOjbXN+9t1TAXjhAz3YXkSkOtEeEeQAS5xzvYAl3nw4B51zWd5rUkj5/cCvveW/Bq6MMh4Arj3nZAAeWPQpzrlYrFJE5IQVbSKYDMz1pucCU2q6oJkZcC4wry7LVyX0BrOesxZSUuo4eLgkFqsWETnhRJsIOjnndgB47x0jtEsxs1wze8/Myn7s2wF7nXNlQ4YWAF3DLw5mNtNbR25hYWG1gT1zxdDg9Mk/X0ifX7xOaamODkREKqo2EZjZYjPLC/OaXIvP6e6cywYuBR40s5MBC9Mu4i+1c+5x51y2cy67Q4cO1X7g2b07MLRnWrmyAXe+ydGSUs771TLWFGiAOhERqEEicM6Ncc71D/OaD+w0s3QA731XhHVs9943AcuAQcBuoI2ZlY0JkQFsj3qLQvz5ymHl5vcXH2Xsr1fwWeEBJj38Du9u3B3LjxMRiUvRnhp6FZjhTc8A5ldsYGZtzaypN90eGAWsdYFe3KXA1KqWj0aTpAQ23jOejfeMD5ZtDnm28aVPvA/AkZJSMnMWkJmzIJYfLyISF6JNBHOAsWaWD4z15jGzbDN7wmvTB8g1s48I/PDPcc6t9epuBW4ys40E+gz+GGU8lSQlJpCUmMD0Yd3D1mfmLKDX7Ndi/bEiInHD4vHyyuzsbJebm1urZZxzPPnOFv7+0XbunNSPyb97J2y7Fk0S+eS/xsUiTBGRRsXMVnn9teWcUHcWV8XMuPLMnrzy41EM7NaGq885KWy7A4dLOFB8lIOHS8jMWcDI+5bUc6QiIvXLN4mgolnj+7Dp3gm8m3Nupbp+t7/B3JVbANi+LzBMxfzV27j40ZW6QU1ETji+OTVUla8OHKZNs2QOl5Ry2m2vV9n2kqHduPd7pxO4H05EJH74/tRQVdJaNCEhwUhJTqy27fMfbKXnrIU88fYmfrM4vx6iExE5vpQIKvhg9nn81+R+1ba7e8E6fr34Uz7dWRQ8XfRxwT5yt3x1vEMUEYkpPeC3go4tU/jhiEwOHi6hZ/sWnN+vM0vW7WT3/mK+f0Z3Tr/9DYqKjwbbn//rFQAkGJSNYHHP9/qzu+gwPx3Tq9L6S0sddy1YyyVDu9O7U8t62SYRkaqoj6AObv7rR+wqKmbFp1WPeTR1SAZdWqdQsPcgd0zqx/gH3+a2C/pyzZ9XAbDg+jPp16V1fYQsIhKxj0CJIAr/9fe1PPnO5qjWsWXOxBhFIyJSNXUWHwe/uLAvW+ZMZOH1Z9V5HTOfySUzZwGv531Zqe7NT74sN2Lq0ZJSBt+1iMycBfzyjQ11/kwRkVA6IoiRXUWHaNOsCZ9s38flT/+TBy4eyBVP59I+tSm79xfXaB2f3TuBRO9ZCrfPz2Puys8B+O+LBrDnwGHuf319ufav/HgUPdu1oHXz5NhujIickHRqqAEsWruT7B5tWbfjGw4cLmFs305cNTeXts2Tuf+iAZz084Xl2k/O6sJvpg3iaEkpU37/DnnbvqnR50Q6vbR2+ze0bp5M1zbNgmU7vznExwX7MIPz+nSq+8aJSNyJlAh01dBxNLZv4Id25Cntg2VPzKi0D4Lmr97Ory/O4uUPt1WZBF6/4SzGPfh2xPrL/vg+b+cHhthOMNh037FEcc2fV/HhF8eexaA+ChFRH0ED+tHITADWhQxy9/7mr7hl3prg/LQzugWnbxl3Ku/NOo/TOrciu0fbYPm+g0fKrbcsCcCxS1qXrt9FZs6CckkA0JAZIqJE0JDumNSPLXMm0qxJIndOCtzEdskf3gvWr79rHHMuGsB1o0/mnN4duG70KXRunQLAvGtHcveU/gD8NXdrlZ+zqXA/lz/9z7B1p9/xJt8cOhK2TkT8QYmgkfjhiB7l5ts0Tw4OeXHLuNOYG/IM5jKDurcBAnc5l/1lXxLmuczn/mp5uflff38gL149Agg8tW3Gkx/UOt6xDywPHmXogT4i8U2JoJEwM2aEJIPVvzi/2mVCb0brOWshz6zcwsleB/T15/UiP+TJbAB3TurHx3ecz/cGZdCjXfNgeUbb5tTGw2/lk7+r/FFGaZgEJCLxQZ3Fjcidk/tz2YgenNKxbkNP/GL+J8HpXh1TSU4sn+cvG96DBO/y1E6tUoLlf/9oO7+9ZBAABV9/y5n3L2XigHR+d+lgDhQf5aEl+Qw/qR0v5m7ltTD3OwAU7i8ut04RiR86ImhkapsE/hjhKqQhXmfy4pvOAaB5k8RgEigTesXQ3He3APDY8k0ALFizg+KjJfS7/Q0eW7GJy5/+Z8QkAPBZ4f5axS0ijYcSQZw7r08nHpk+uFzZpnsn0MW7d+CUjqlsvm8CeXd8N+zyN3gD493+auBo4mhpabDu1P+M/GyG/3dWTwAGdgv0U2zcVfdEsKvoELu+OVTn5UUkOkoEJ4Dxp6czY0QPXpg5nC1zJlb6y9/MKpWV+cm5x0ZInflMLs9/UPUVSADd0poxe2JfNt83gVeuG0nzJonk79zPAW9U1tJSx2V/fJ+Cr7+ttGzFzuzSUsfQe5Yw9F49ElSkoejOYuH7j63k/c3ln6NwwYB0/rFmBwAf3X4+A+98M+JoqaFXDc08+yT+mruVr789dknqS9eOYHD3tvScFejILjsllbvlK6Y+ujLYbv1d42r0cCARqRvdWSwRTRyQXi4RpCQn8PClg3loWuCPhIQEq/EdyI+v2FSp7KJHVnJ612MJ5LHln3HRkIxySQCgsKiYbmm1u4JJRKKnU0PCZcN7sPTm0cH5edeMBAIJINIppVChdz9H8vG2fcHp+15bT/bdiyu1+bKafoLSUseuIvUliMSaEoFgZvRs34KF15/FJ3d+l/5da/ewnLum9OeOC/uWK/vzlcPYcPe4CEscc9PY3rx549kA7NhX9Y/8b9/ayNB7ljBqzlu1ik9EqqZTQxLUt0urOi2XnJjAj0b1pMTBXf9YC8CZvQID7W2ZM5Gb/rKalz/cxgszh7N+xzfc8fe1wWV/cu4pwUd/Xv/8h0wa2AWA/J1F5Lz8Mas+/zrYtmPLpgBs23sQgI+27mXR2p2kpiTRokkil43IrFP8In6nRCAxc8WoTF75cBs3je1drvyB72fxwPezAMjq1oaH3tqIAW/f+h3MjJZNj/1v+MWeb0lNSWKs9yzoULuKjj3X4SfPf8jfP9pern7YSe30HGiROtBVQ9IoxGK8ou+c2oGnLq88JpOIBByXR1WaWZqZLTKzfO+9bYR2JWa22nu9GlL+tJltDqnLiiYeiV+r/nNM2PIbx/Tm8cuGML5/ZwCuPvukiOtYuqEwOPheaamr1fhH+w4eCd4HIeI30Z4aygGWOOfmmFmON39rmHYHnXORfuR/5pybF2UcEufapTatVBZ6yer5/ToHp8efns6U373Dby8ZxIVen0LZEcW9C9cxe2Lf4NPf3s05N3iXdSRHS0oZeOeblT5TxC+ivWpoMjDXm54LTIlyfeJj15xzcnD6uauGRWyX1a0Nm+6dEEwCAJcO6w4EnvJWFPJ8hZFz3goeJfSavZAJvzn2ZLdDR0rIzFnAKbNfC5YdKTk2xIaIX0TVR2Bme51zbULmv3bOVTo9ZGZHgdXAUWCOc+4Vr/xpYARQDCwBcpxzYZ/0bmYzgZkA3bt3H/L555/XOW5pnEpKHb9bupExfTrV6QqmsqOClk2TglciQeCoYGSFS06vOedkHl3+WaV1/OMnZ9b68lmReFHnPgIzW2xmeWFek2vx+d29D78UeNDMyv70mwWcBpwBpBH+tBIAzrnHnXPZzrnsDh061OKjJV4kJhjXn9erzpexlilLAlefE+hP+OUbGyq1CZcEAA4eKQlbfuhICTf+ZTVrt0d+lrRIvKo2ETjnxjjn+od5zQd2mlk6gPe+K8I6tnvvm4BlwCBvfocLKAaeAnTJh8TMvw3KAODlD7dFbHPuaR0B6O4NbXHfwnVh25122+v87cNtTHjo7XLls15eQ2bOgnKno0TiTbR9BK8CM7zpGcD8ig3MrK2ZNfWm2wOjgLXefFkSMQL9C3lRxiM+9tK1I4PT7806j1M7t6xQP4ILBqQH51umJPHkj85g830T+Os1gUd3/uuLvZXWW3y08lHCOxt3k5mzIDha6+l3vKlHdkrciraPoB3wItAd+AL4d+fcV2aWDVzjnLvKzEYCjwGlBBLPg865P3rLvwV0AIxAH8I1zrlqB7bXfQQSyf7ioyQlWHAU05JSx9V/yqV3p5bcMu60YLt9B4/QullyuWVj8UP+H2N785PzelXfUKQBROoj0A1lIp7Ln/qApRsKo17Ppnsn1GiwPpH6dlxuKBM5kdx/0YAq628+v/zQGc9dNYx/3TaWj24/n1njjx1tvLdpT8R1LP+0kPydRdEFKhJjSgQino6tUvjzlcPoltaMa0efHOxPuP68XnRLa8aMkZk8+aPAH1MvXTuCkae0J61FE1o3S+bqc07mqR+dAcClT7wf9q7mvd8eZsaTH5QbR+lA8VEWrNnB4aO6f0Eajk4NicSIcy74FDaAMX06sXjdTp67ahif7iwqN+rqljkTy7Xv37UVXx84wi3jTmVyVtd6j138QX0EIvVgxH1Lqn2uAsArPx7FlN+9E7H+07vH0yRJB+wSW+ojEKkHb9/yHSac3rnadlUlAYBNu6u9eE4kZpQIRGIoKTGB314ymKGZaQzNTOPnE451Iq+cdS6/nz640jLLfza6UtmSdeXvzQw9cp/18hpe+XAbM578oFzH9K5vDrH800LWFFS+F0KkKjo1JFKP8ncWBTuLl908mrTUJrRKSea6Z1dxft/ODO2ZFhwXqWwk1LL7G5bePJpWKUkMqfC852vOOZnd+4uZt6ogWLbg+jPp10VjJkl56iMQaSR2FR2iQ2pTAjfUlxfagfzWf5xDakoSQ+9ZUqfP0ZDaUpH6CEQaiY4tU8ImAQAz4/9/5xQAPt/zbZ2TAMA3dRz/aNyDK8jMWcB+PajHN5QIRBqZy0dlAvDm2p0R24zv35kBGYFTP9ecczJn9+7A0Mw0tsyZSJfWKQB8vvvbaj/rtY93kJmzgI279lN06AiZOQtY/2Xghrftew9GuSUSL/TwepFGpuxpbc9/8EWwbPN9EzAzduw7SIfUpiQlBv6Gc85VOrq4fVI/rv7TKi58+H9Z/rPR7N5fTP+urfliz7ektWgSXH/o2ErTHn+P3fvLPwrk28Phh+SWE48SgUgjl3fnd4M/9umtyz92M9wppvapTYLT5/zPskr1W+ZM5Ln3vyhXVjEJAHx94HBdwpU4pEQg0silNq3dP9MhPdLo1TGV/F3h70Wo6Siredv20aZ5Mv26tK50c9szK7ewu6iYG8f2jtjfIfFDfQQijVDZ8xEmhTyXuTYW3XROjdp1bVP+CGPzfRO4YUxgGO1fLfqU7/3+XXr/57FnOjvn2LO/mF/M/4SH3trImoJ95Zbfs7+YL/ZU3zchjYuOCEQaoTO8jt9oPHbZEK7+0yo6tWpKj3Yt+GDzV+Xq1981jpTkREpKHcPuXczcK4ZiZtwwpjcPLs4v1/bw0VKaJCWUG0sJYMueAwzsFnxsebl7HCZndeHB72fpiCEO6D4CEZ84dKSE0257HYB/zh5Dh5ZNI7Ydes9idhUd6zcY3Im5xK4AAAYNSURBVL0Nz1w5jP63v1GpbfMmifx8Qh8GZrThwof/t1yd7mVoXHRDmYjUWKweu6lE0LjohjIRqbGlN4+usq6qH/hl3rIPXDwwxlHJ8aI+AhGppGf7Frx549l0T2vO5t0HGP+bt4N1me2aB9ts3n2g0rKZ7VvoSCDO6IhARMLq3aklKcmJ9O7UMlj20CWDgp2/z1wxlCZJCdw9pX+w/rWfnlXvcUr04rKPwMyKgA0NHUcU2gO7GzqIKMX7NsR7/BD/2xDv8UP8bUMP51yHioXxempoQ7gOj3hhZrnxHD/E/zbEe/wQ/9sQ7/HDibENoFNDIiK+p0QgIuJz8ZoIHm/oAKIU7/FD/G9DvMcP8b8N8R4/nBjbEJ+dxSIiEjvxekQgIiIxokQgIuJzcZUIzGycmW0ws41mltPQ8ZQxs25mttTM1pnZJ2b2U688zcwWmVm+997WKzcze8jbjjVmNjhkXTO89vlmNqMBtiXRzD40s3948z3N7H0vnr+YWROvvKk3v9GrzwxZxyyvfIOZfbceY29jZvPMbL23L0bE2z4wsxu9/4fyzOx5M0tp7PvAzJ40s11mlhdSFrPv3cyGmNnH3jIPWYyHM40Q//94/x+tMbO/mVmbkLqw322k36dI+69Rcc7FxQtIBD4DTgKaAB8BfRs6Li+2dGCwN90S+BToC/w3kOOV5wD3e9MTgNcAA4YD73vlacAm772tN922nrflJuA54B/e/IvANG/6UeBab/o64FFvehrwF2+6r7dvmgI9vX2WWE+xzwWu8qabAG3iaR8AXYHNQLOQ7/5HjX0fAGcDg4G8kLKYfe/AB8AIb5nXgPH1EP/5QJI3fX9I/GG/W6r4fYq0/xrTq8EDqMXOGgG8ETI/C5jV0HFFiHU+MJbA3c/pXlk6gRvhAB4DLglpv8GrvwR4LKS8XLt6iDsDWAKcC/zD+4e3O+QfRHAfAG8AI7zpJK+dVdwvoe2Oc+ytCPyIWoXyuNkHBBLBVu/HMMnbB9+Nh30AZFb4IY3J9+7VrQ8pL9fueMVfoe57wLPedNjvlgi/T1X9G2pMr3g6NVT2j6RMgVfWqHiH54OA94FOzrkdAN57R69ZpG1p6G18ELgFKPXm2wF7nXNHw8QTjNWr3+e1b6htOAkoBJ7yTm09YWYtiKN94JzbBvwS+ALYQeA7XUX87INQsfreu3rTFcvr0xUEjkSg9vFX9W+o0YinRBDuvGCjuvbVzFKBl4AbnHPfVNU0TJmrovy4M7MLgF3OuVWhxVXE09i2IYnA4f0jzrlBwAECpyQiaWzx451Hn0zglEMXoAUwvop4Gt021EBtY27QbTGz2cBR4NmyogjxNMr4ayqeEkEB0C1kPgPY3kCxVGJmyQSSwLPOuZe94p1mlu7VpwO7vPJI29KQ2zgKmGRmW4AXCJweehBoY2ZlY1KFxhOM1atvDXxFw21DAVDgnHvfm59HIDHE0z4YA2x2zhU6544ALwMjiZ99ECpW33uBN12x/LjzOqwvAKY777xONXGGK99N5P3XaMRTIvgn0MvrgW9CoHPs1QaOCQhcCQH8EVjnnHsgpOpVoOzqhxkE+g7Kyn/oXUExHNjnHT6/AZxvZm29vw7P98qOO+fcLOdchnMuk8B3+5ZzbjqwFJgaYRvKtm2q19555dO8K1p6Ar0IdPYd7/i/BLaa2ale0XnAWuJoHxA4JTTczJp7/0+VbUNc7IMKYvK9e3VFZjbc+05+GLKu48bMxgG3ApOcc99W2K5w323Y3ydvf0Taf41HQ3dS1OZF4IqDTwn0zs9u6HhC4jqTwOHeGmC195pA4PzgEiDfe0/z2hvwO287PgayQ9Z1BbDRe13eQNszmmNXDZ1E4H/0jcBfgaZeeYo3v9GrPylk+dnetm0gxld4VBN3FpDr7YdXCFx9Elf7ALgTWA/kAX8icHVKo94HwPME+jSOEPjL+MpYfu9Atvd9fAY8TIULAo5T/BsJnPMv+/f8aHXfLRF+nyLtv8b00hATIiI+F0+nhkRE5DhQIhAR8TklAhERn1MiEBHxOSUCERGfUyIQEfE5JQIREZ/7P6k60hy70mCHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(20, 1e-6, start_epoch=0)#, moms=(0.75,0.70))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:21:05.284490Z",
     "start_time": "2019-08-21T02:21:04.781955Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwAAAAEGCAYAAADSR/K6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzdd3hUZdrH8e+dTkISSKGlQOi9hlAsWFZFVLBRLEhzdYvrrm5Rt7ruurvua9l11bXSFAXEhr1iA4EEQodACCUJLbRQQvr9/jGDG2NCBkhyMjP357rmYnLmnOf8Jspw7jlPEVXFGGOMMcYY4x8CnA5gjDHGGGOMaTxWABhjjDHGGONHrAAwxhhjjDHGj1gBYIwxxhhjjB+xAsAYY4wxxhg/EuR0AF8RFxenHTp0cDqGMcacthUrVuxX1XinczQm+8w2xnir+vjMtgKgnnTo0IGMjAynYxhjzGkTkR1OZ2hs9pltjPFW9fGZbV2AjDHGGGOM8SNWABhjjDHGGONHrAAwxhhjjDHGj1gBYIwxxhhjjB+xAsAYY4wxxhg/0qAFgIiMFJEsEckWkXtreD1UROa5X18mIh2qvHafe3uWiFxWV5sikuJuY4u7zZBTnUNEYkVkkYgcE5EnquUaJCJr3cc8LiJS378bY4wxxhhjnNBgBYCIBAJPApcDPYEbRKRntd2mAYdUtTPwGPCQ+9iewASgFzASeEpEAuto8yHgMVXtAhxyt13rOYBi4A/Ar2qI/1/gNqCL+zHyTH8PxhhjjDHGNCUNuQ5AGpCtqjkAIjIXGANsqLLPGOB+9/MFwBPub9vHAHNVtQTYJiLZ7vaoqU0R2QhcBNzo3meWu93/1nYOVT0OfC0inauGFpG2QJSqfuP+eTZwNfD+Wf02jM+qqFS2Fhwja89R9h4p5nhJBSIQGhTgegQHEhkWRExECHHNQ4mJCCEmPISAALuxZIy3KCot58lF2UQ3C6ZFeAhdW0fSvU0kYcGBTkczxpjT1pAFQAKQW+XnPGBIbfuoarmIFAKx7u1Lqx2b4H5eU5uxwGFVLa9h/9rOsf8UufNqOfd3iMhtuO4UkJycXEtzxhepKsu2HWR+Ri6fZxVw8HjpaR0fEhRAckw47WPCaR8bQadWEfRqF20XFMY0UYeLynj6ixwqKvXbbSFBAZzXOY6r+rVjVJ+2hATZsDpjjHdoyAKgpq831cN9atte06frqfb3NIcnmb6/UfVZ4FmA1NTUU7VpfISq8snGfTz68WY27j5CVFgQF3VvxXld4unRNoqEls2ICHFdwJdWVFJSVklJeSVHisvYf6yEg8dL2X+0hN2FxWw/cJwdB4pYsvUAJ8oqAAgMEDrHN6dPYjRpKTEM6xhLYstm2DAUY5zVrkUzsh+8nKLSCgqOlrBpz1GWbTvAR+v38ummVfztvY3cdn5HbhnWwQoBY0yT15AFQB6QVOXnRGBXLfvkiUgQEA0crOPYmrbvB1qISJD7LkDV/Ws7x6lyJ9aR2/ihbfuP87s31rJk6wE6xkfw0HV9GNM/odZv7IMCAwgPcT1vEx1G19aRNe6nquQdOsH6XUdYv6uQdfmFfLpxLwtWuG5EJbRoxpCOMVzUvRXnd40nKiy4Qd6fMebURISI0CAiQoPoEBfByN5t+OOVPflyy36e+zKHv767kZeW7uCBMb05v2u803GNMaZWDVkApANdRCQFyMc1qPfGavssBCYB3wDXA5+pqorIQuBlEXkUaIdrIO5yXN/Of69N9zGL3G3Mdbf51qnOUVtoVd0tIkdFZCiwDLgF+M/Z/SqMN1NVXlmey1/e2UBwoPDAmF7ckJZMcGD9fMsnIiTFhJMUE87I3m0AqKxUtuw7xrJtB1iac4DPNu3j9ZX5BAcKQ1Ji+UGPVlzepy2to8LqJYMx5syICCO6xjOiazyfZ+3jL+9s4Jbpy5k0rD33jephXfqMMU2SnOJa+OwbFxkF/AsIBKar6oMi8gCQoaoLRSQMeBEYgOtb+QlVBvj+DpgKlAO/UNX3a2vTvb0jrov/GCATuFlVS+o4x3YgCggBDgOXquoGEUkFZgLNcA3+/dmpigZwdQHKyMg421+ZaWJKyiu47/W1vL4yn3M7x/Hw2H60iW78i+7yikpW7jzMpxv38snGvWwtOE6AwPBOcYzp346RvdsQaXcGzBkSkRWqmup0jsbUUJ/ZxWUV/PODLKYv3kbvhCiev2WwI58ZxhjfVR+f2Q1aAPgTKwB8z4FjJdz+4goydhzi7ku6cseFnZvMzD3Z+46xcFU+b67axc6DRYQGBTCydxsmDm3PoPYtbcyAOS1WANS/Tzfu5c5XMokIDeKFSYPpkxjdYOcyxvgXKwCaECsAfMu+o8Xc9Nwydh4s4pFx/biybzunI9VIVVm58zBvZubzZmY+R0vK6d4mkpuGtueaAQk0D23IXn7GV1gB0DCy9hxl6sx0jpwoY+bUwQxqH9Og5zPG+If6+My2qQqMqWbvkWImPLuUvEMnmDklrcle/IOr//Gg9i35y9W9Wfrbi/n7tX0IEOEPb65j2N8+5e/vb2TfkWKnYxrjl7q1iWTBj4cRFxnKxBeWsyzngNORjDEGsALAmO8oPFHGxBeWsbewmFlT0xjWKdbpSB6LCA3ihrRk3r3zXF7/yXBGdIvnuS9zOPehRdz3+lq27T/udERj/E7b6GbMu20o7Vo0Y9qsDNbvKnQ6kjHGWAFgzEkl5RXcNjuDbfuP89ykVNJSvPN2vYgwMLklT9w4kEW/uoCxqYm8tjKPix75nF/MzWS7FQLGNKpWUWG8OC2NqLAgJs9IJ/dgkdORjDF+zgoAY3D1pf/l/NUs23aQh8f2Y3inOKcj1Yv2sRE8eE0fFt9zEbef34kP1+/l4ke/4L7X17Dr8Amn4xnjN9pGN2PW1DRKyyuZNH05hSfKnI5kjPFjVgAYA/z3i628s2Y394zszpj+CU7HqXfxkaHce3l3vvjNBUwc2p7XVuRzwcOf85d3NlBYZBcixjSGLq0jee6WVHYeLOKueauorLRJOIwxzrACwPi9r7fs5+EPs7iyb1t+NKKj03EaVKvIMO4f3YtFv76Aa/onMGPxNkY8vIjZ32ynvKLS6XjG+Ly0lBj+dFVPPtu0j399usXpOMYYP2UFgPFr+YdPcOfcTDq3as5D1/X1m/nzE1o046Hr+/LunefRs20Uf3xrPaMe/4qvthQ4Hc0Yn3fz0PaMHZTI459u4bNNe52OY4zxQ1YAGL9VUan8Ym4mpeWVPH3zICL8cM78Hm2jmHPrEJ6ZOIjiskomvrCc21/MYHehjQ8wpqGICH+5ujc92kbx61fXUHC0xOlIxhg/YwWA8VvPfLmV9O2H+PPoXnSMb+50HMeICJf1asPHd5/Pry/rxhebC7jk0S+ZuXgbFdZH2ZgGERYcyOMT+nOspJxfL1iNLcppjGlMVgAYv7Quv5DHPt7MqD5tuHag7w36PROhQYH89MLOfPSLEQxIbsH9b2/g2v8uYcOuI05HM8YndWkdye+v6MHnWQXMXLLd6TjGGD9iBYDxO8VlFdw1bxUtw0N48Oo+ftPv31PJseHMnprGvyf0J+9gEVc98TUPf5hFabkNEjYNR0RGikiWiGSLyL01vN5eRD4VkTUi8rmIJFZ7PUpE8kXkicZLffZuHtqei7q34h/vb7I1OowxjcYKAON3nlyUzZZ9x/jn9X1pGRHidJwmSUQY0z+BT385gqv7J/DEomyufnIxm/bY3QBT/0QkEHgSuBzoCdwgIj2r7fYwMFtV+wIPAH+v9vpfgC8aOmt9ExH+dk0fQgIDuO/1tdYVyBjTKKwAMH5l896jPP3FVq4dkMAF3Vo5HafJaxEewiPj+vHcLansO1rMVf/5mqc+z7axAaa+pQHZqpqjqqXAXGBMtX16Ap+6ny+q+rqIDAJaAx81QtZ61yY6jPtG9eCbnAPMS891Oo4xxg9YAWD8RmWlct/ra2keGsTvrujhdByvcknP1nx01wgu6dmaf36Qxdinl7DjgHVXMPUmAah65Zvn3lbVauA69/NrgEgRiRWRAOAR4Nd1nUREbhORDBHJKChoWlPeThicxJCUGB58byN7jxQ7HccY4+OsADB+45X0nazYcYjfXdGT2OahTsfxOjERITx540D+PaE/2fuOccXjX/P26l1OxzK+oaaBONVvM/0KGCEimcAIIB8oB34CvKeqdX51rqrPqmqqqqbGx8efbeZ6FRAg/OO6vpSUV/LXdzc6HccY4+OsADB+4cCxEv7x/iaGd4rlOpv154ydHBvw3s/Po2vr5vzslUzue30NJ0ornI5mvFsekFTl50TgO9Wlqu5S1WtVdQDwO/e2QmAYcIeIbMc1TuAWEflHo6SuZylxEfxoRCfeXr2LZTkHnI5jjPFhVgAYv/Dox5spKq3gz6N72aw/9SCxZTjzbh/GTy7oxNz0XEY/8TVZe446Hct4r3Sgi4ikiEgIMAFYWHUHEYlzd/cBuA+YDqCqN6lqsqp2wHWXYLaqfm8WIW/x4xGdSGjRjD8tXE95hc28ZYxpGFYAGJ+3cfcRXlm+k4lD29OldaTTcXxGcGAAvxnZndlT0zhUVMboJ75mfoYNYDSnT1XLgTuAD4GNwHxVXS8iD4jIaPduFwBZIrIZ14DfBx0J28CahQTyuyt6sGnPUV5ZvtPpOMYYHyU25Vj9SE1N1YyMDKdjmGpUlZueX8aG3Uf4/FcX0CLcpv1sCPuOFnPXvFUszj7ATUOS+eNVPQkNCnQ6lvGQiKxQ1VSnczSmpvyZffJza/2uIyz61QXE2HTFxpgq6uMz2+4AGJ/20Ya9LNl6gLt+0NUu/htQq8gwZk1J40cjOjFn2U4mPLuUPYU2k4kxZ0JEuH90L44Wl/Gfz7Y4HccY44OsADA+q7S8kr+9t5EurZpz05Bkp+P4vKDAAO69vDtP3TSQrD1HufI/X9lARmPOUNfWkYxLTeKlpTvIPVjkdBxjjI+xAsD4rHkZuew4UMR9o7oTFGj/qzeWUX3a8tZPzyEqLJgbn1/GjMXbbHVTY87AL37QlcAA4ZGPspyOYozxMXZVZHzSidIK/vPpFlLbt+RCW/G30XVpHcmbd5zDhd1a8ee3N/C7N9dRZjOaGHNa2kSHMfWcFN5ctYt1+YVOxzHG+BArAIxPmvXNdvYdLeE3I7vbtJ8OiQoL5tmJg/jxBZ14edlOJs9YTmFRmdOxjPEqt4/oRIvwYB76YJPTUYwxPsQKAONzCk+U8d/Pt3JBt3jSUmKcjuPXAgKEe0Z25/+u78vybQe55qnFbNt/3OlYxniN6GbB3HFhZ77asp8l2fudjmOM8RFWABif89yXORSeKONXl3ZzOopxG5uaxJxbh3KoqJSrn1zMkq12IWOMp24e2p42UWE89slmG09jjKkXVgAYn7L/WAnTF2/jyr5t6Z0Q7XQcU0VaSgxv/fRc4iNDueWF5SxYked0JGO8QlhwID+5sBPp2w+xZKvNrGWMOXtWABif8vxX2zhRVsFdl3R1OoqpQXJsOK//ZDhDOsbwq1dX8+SibPtG0xgPjEtNok1UGP+yuwDGmHpgBYDxGYeOl/LiN9u5sm87OsU3dzqOqUVUWDAzJqcxpn87/u/DLP741noqKu2CxphTsbsAxpj6ZAWA8RkzFm/jeGkFd1zY2ekopg4hQQE8Nq4/t5/fkReX7uDHL62guKzC6VjGNGl2F8AYU1+sADA+4UhxGTOWbOeyXq3p1ibS6TjGAwEBwn2jevCnq3ry8ca93PT8Mg4XlTody5gmq+pdgMXZdhfAGHPmGrQAEJGRIpIlItkicm8Nr4eKyDz368tEpEOV1+5zb88SkcvqalNEUtxtbHG3GXIW57hLRNaLyDoReUVEwur7d2Pq1+wl2zlaXM7PLuridBRzmqack8ITNwxkbV4h1/13CbsOn3A6kjFN1sm7AE8s2uJ0FGOMF2uwAkBEAoEngcuBnsANItKz2m7TgEOq2hl4DHjIfWxPYALQCxgJPCUigXW0+RDwmKp2AQ652z6TcyQAdwKpqtobCHTvZ5qo4yXlvPD1Ni7q3spm/vFSV/Rty+xpaew7UsLYp79hu60VYEyNwoIDufW8FJbmHCRz5yGn4xhjvFRD3gFIA7JVNUdVS4G5wJhq+4wBZrmfLwAuFteyrWOAuapaoqrbgGx3ezW26T7mIncbuNu8+gzPARAENBORICAc2FUPvw/TQF5auoNDRWXccZH1/fdmQzvG8vIPh1JUWs7YZ75h054jTkcypkmakJZMVFgQT3+x1ekoxhgv1ZAFQAKQW+XnPPe2GvdR1XKgEIg9xbG1bY8FDrvbqH6u0zqHquYDDwM7gd1Aoap+VNMbFJHbRCRDRDIKCgpq/UWYhlNaXskLX29jeKdYBia3dDqOOUt9EqOZf/swAgTGP7OUVbmHnY5kTJPTPDSIScM78NGGvWwtOOZ0HGOMF2rIAkBq2FZ92oLa9qmv7ad9DhFpievuQArQDogQkZtr2BdVfVZVU1U1NT4+vqZdTANbuHoX+46WcPuITk5HMfWkS+tIFvxoONHNgrnpuaW2arAxNZg0vAMhgQE8+0WO01GMMV6oIQuAPCCpys+JfL8rzbf7uLvbRAMHT3Fsbdv3Ay3cbVQ/1+me4wfANlUtUNUy4HVg+Gm8b9NIVJXnvsyhW+tIzu8S53QcU4+SYsJ59UfDSGjZjMkz0vlkw16nIxnTpMQ1D2VcahKvZ+axp7DY6TjGGC/TkAVAOtDFPTtPCK6BtAur7bMQmOR+fj3wmbomN14ITHDP4JMCdAGW19am+5hF7jZwt/nWGZ5jJzBURMLdYwUuBjbW0+/E1KMvNheQtfcoPzy/I67/VMaXtI4KY95tw+jeJpIfvbSCd9fsdjqSMU3Kbed3pFJh+uJtTkcxxniZBisA3P3t7wA+xHUBPV9V14vIAyIy2r3bC0CsiGQDdwP3uo9dD8wHNgAfAD9V1Yra2nS3dQ9wt7utWHfbZ3KOZbgGC68E1uL6HT1b778gc9ae+yqH1lGhjO7XzukopoG0jAhhzq1D6J/UgjvnZvL2ahuPb8xJSTHhXNGnLXOW7qDwRJnTcYwxXkRsNcH6kZqaqhkZGU7H8Bvr8gu58j9fc+/l3fmR9f/3ecdKypk6I52MHQd5bHx/xvSvPp+AORsiskJVU53O0Zh85TP75Gfh70b14Ifnd3Q6jjGmEdTHZ7atBGy80vNf5RAREsgNaclORzGNoHloEDOmDCa1Qwx3zVvFm5n5TkcypknonRDNkJQYZi7ZTnlFpdNxjDFewgoA43XyD5/g7TW7mZCWTHSzYKfjmEYSERrEzCmDGZISy93zV/H6yjynIxnTJEw9N4X8wyf42AbLG2M8ZAWA8TqzlmwHYMo5HRzNYRpfeEgQ0ycPZlinWH756mpezcit+yBjfNwPerQmKaaZDQY2xnjMCgDjVYpKy5m7fCcje7UhsWW403GMA5qFBPLCpMGc2zmO37y2hvlWBBg/FxggTB6eQvr2Q6zJs8XzjDF1swLAeJU3M3dxpLicScM7OB3FOCgsOJDnbknlvC7x3PPaGhsTYPzeuNRE11iZxdudjmKM8QJWABivoarMWrKdHm2jGNyhpdNxjMPCggN5duIghrrHBLy31tYJMP4rMiyYsamJvLNmF3uP2MJgxphTswLAeI2lOQfJ2nuUKcM72MJfBnAVAc9PSmVgckvufCXTVgw2fm3y8A6UVyovLd3hdBRjTBNnBYDxGrOWbKdleDCj+9vCX+Z/ItxThPZqF8VP5qzki80FTkcyxhHtYyP4QY/WzFm2k+KyCqfjGGOaMCsAjFfIP3yCjzbsYfzgZMKCA52OY5qYyLBgZk8dQudWzbltdgZLtu53OpIxjpgyvAMHj5dalzhjzClZAWC8wslb2jcPtYW/TM2iw4N5cVoayTHh3Dorg4ztB52OZE6DiIwUkSwRyRaRe2t4vb2IfCoia0TkcxFJdG/vLyLfiMh692vjGz990zGsUywd4yOY/Y11AzLG1M4KANPkFZdVMHf5Ti7taVN/mlOLbR7KnB8OoU1UGJNnpLM616ZE9AYiEgg8CVwO9ARuEJGe1XZ7GJitqn2BB4C/u7cXAbeoai9gJPAvEWnROMmbHhFh4tD2rMo9zNq8QqfjGGOaKCsATJO3cNUuDhWV2dSfxiOtIsOY88MhtIwIZtKM5Wzee9TpSKZuaUC2quaoaikwFxhTbZ+ewKfu54tOvq6qm1V1i/v5LmAfEN8oqZuoawcm0iw40AYDG2NqZQWAadJUlVnfbKdb60iGdoxxOo7xEm2jmzFn2lBCAgOY+MIycg8WOR3JnFoCUHVFtzz3tqpWA9e5n18DRIpIbNUdRCQNCAG2NlBOrxDdLJirB7TjrdX5FBaVOR3HGNMEWQFgmrTVeYWs33WEicPa29Sf5rQkx4bz4rQhFJdVMvGFZew7anOjN2E1/eXWaj//ChghIpnACCAfKP+2AZG2wIvAFFWtrPEkIreJSIaIZBQU+PZsUTcPbU9xWSULVuY5HcUY0wRZAWCatDlLdxAeEsgYm/rTnIFubSKZMWUwe4+UcMsLyyk8Yd+GNlF5QFKVnxOBXVV3UNVdqnqtqg4AfufeVgggIlHAu8DvVXVpbSdR1WdVNVVVU+PjfbuXUK920Qxq35KXlu6gsrJ6LWWM8XdWAJgmq/BEGW+v2cWY/glEhgU7Hcd4qYHJLXn2lkFsLTjGtJnpnCi1+dGboHSgi4ikiEgIMAFYWHUHEYkTkZP/Zt0HTHdvDwHewDVA+NVGzNzkTRzanm37j7Nk6wGnoxhjmhgrAEyT9WZmPsVlldw0xKb+NGfnvC7x/HvCAFbuPMSP56ygtLzGHiLGIapaDtwBfAhsBOar6noReUBERrt3uwDIEpHNQGvgQff2ccD5wGQRWeV+9G/cd9A0Xd6nDTERIby4dLvTUYwxTYwVAKZJUlXmLNtBv8RoeidEOx3H+IBRfdryt2v68HlWAXfPX0WFdYtoUlT1PVXtqqqdVPVB97Y/qupC9/MFqtrFvc+tqlri3v6Sqgarav8qj1VOvpemIjQokPGDk/h4w152HT7hdBxjTBNiBYBpklbsOMTmvce40b79N/VoQloy913enXfW7OaPb61D1YoA49tuTEtGgbnLdzodxRjThHhUAIjIuSIyxf08XkRSGjaW8XcvL9tJZGgQV/Wzwb+mft0+ohM/vqATc5bt5LGPNzsdx5gGlRQTzvld4pmfkUd5hXV9M8a41FkAiMifgHtwDboCCAZeashQxr8dOl7KO2t3c83ABMJDgpyOY3zQby7rxvjUJB7/LNsWSzI+74a0ZPYcKeaLzb499akxxnOe3AG4BhgNHIdvV1qMbMhQxr+9tjKP0vJK6/5jGoyI8OA1vbm4eyv++NY6Ply/x+lIxjSYi3u0Iq55KK9YNyBjjJsnBUCpujrKKoCIRDRsJOPPVJWXl+9kYHILureJcjqO8WFBgQE8ceNA+iW14M5XMknfftDpSMY0iODAAMamJvLZpn3sKbQF8YwxnhUA80XkGaCFiPwQ+AR4vmFjGX+1NOcgOQXHuWlIe6ejGD/QLCSQFyYNJqFFM6bNTGfL3qNORzKmQUwYnESlwqsZuU5HMcY0AXUWAKr6MLAAeA3oBvxRVR9v6GDGP728fCfRzYK5om9bp6MYPxETEcKsqWmEBgcyafpydhfadInG97SPjeCczrHMTc+1lYGNMR4NAn5IVT9W1V+r6q9U9WMReagxwhn/cuBYCR+s2821AxMICw50Oo7xI0kx4cycMpgjxeVMnp5O4YkypyMZU+8mDE4m//AJvsre73QUY4zDPOkCdEkN2y6v7yDGvJGZT1mFcmOaDf41ja9Xu2ienTiInP3H+OHsDIrLKpyOZEy9urRXa2IiQmxNAGNM7QWAiPxYRNYC3URkTZXHNmBN40U0/kBVmZeey4DkFnRpbZNMGWcM7xzHI+P6s3zbQe6aZ6sFnykRSRWRN0RkpfvfjbUiYv9uOCw0KJDrBibw8Ya9FBwtcTqOMcZBp7oD8DJwFbDQ/efJxyBVvbkRshk/kpl7mC37jjE+NcnpKMbPje7Xjj9c2ZP31+3hz2+vt9WCz8wcYAZwHa5/N650/2kcNn5wMuWVyoIVeU5HMcavrcsvZNpM57qc1rrKkqoWAoXADQAi0goIA5qLSHNVtXuIpt68mpFLs+BAG/xrmoRp56aw70gxz3yZQ0KLZtw+opPTkbxNgaoudDqE+b7OrZqTlhLDvPSd3H5+RwICxOlIxvidnQeKmDxjOaFBgRSXVRDdLLjRM3gyCPgqEdkCbAO+ALYD7zdwLuNHikrLeXv1bq7o25bIsMb/S2BMTe4Z2Z2r+rXj7+9v4p01u5yO423+JCLPi8gNInLtyYfToYzLDWlJbD9QxNKcA05HMcbvHDhWwi3Tl1FeqcyamkbrqDBHcngyCPivwFBgs6qmABcDixs0lfEr767ZzbGScsYPtu4/pukICBD+7/q+DO7QkrvnrybDFgo7HVOA/sBI/td99EpHE5lvXd67LVFhQbySbmsCGNOYikrLmToznd2FxbwwKZXOrZo7lsWTAqBMVQ8AASISoKqLcH2w10lERopIlohki8i9NbweKiLz3K8vE5EOVV67z709S0Quq6tNEUlxt7HF3WbIWZyjhYgsEJFNIrJRRIZ58n7NmXk1I4+OcRGktm/pdBRjviMsOJBnJ6aS2KIZt87OIKfgmNORvEU/VU1V1UmqOsX9mOp0KOMSFhzItQMT+XDdHg4XlTodxxi/UFZRyU/nrGRtfiFP3DiQQe1jHM3jSQFwWESaA18Cc0Tk30B5XQeJSCDwJK4pQ3sCN4hIz2q7TQMOqWpn4DHgIfexPYEJQC9c3yA9JSKBdbT5EPCYqnYBDrnbPu1zuI/5N/CBqnYH+gEbPfg9mTOQU3CM5dsPMjY1CRHri2qanpYRIcyckkagCJNnpHPgmM2e4oGlNXzemyZkXGoSpRWVvLXKurcZ09BUld+9sZZFWQX89eo+XNKztdORPCoAxgBFwF3AB8BWPJvNIQ3IVtUcVS0F5rrbqt72LPfzBcDF4roKHHMHms8AACAASURBVAPMVdUSVd0GZLvbq7FN9zEXudvA3ebVZ3IOEYkCzgdeAFDVUlU97MH7NWdgfkYegQHCdYMSnI5iTK2SY8N5flIq+44Wc6utEeCJc4FV7rurNg1oE9SzXRS9E6KYn2HdgIxpaI9+vJn5GXnceXEXbhzSNNY6qrMAUNXjqlqpquWqOgvXN/AjPWg7Aaj6yZLn3lbjPqpajmvWodhTHFvb9ljgsLuN6uc63XN0BAqAGSKS6R7IFuHB+zWnqbyiktdW5nFht1a0inRmEIwxnhqQ3JJ/TxjAqtzD/GKurRFQh5FAF+BSbBrQJmtcahLrdx1hXX6h01GM8VkvLd3Bfz7LZsLgJO76QRen43zrVAuBRbn7yD8hIpeKyx1ADjDOg7Zr6s9R/V/M2vapr+1nco4gYCDwX1UdABwHvjd+AUBEbhORDBHJKCgoqGkXcwqfZxVQcLSEcamJTkcxxiOX9WrDH67oyQfr9/C396xn4CloLQ/ThIzu146QoABbE8CYBvLh+j388a11XNy9FX+9uneT6up8qjsALwLdgLXArcBHwFhgjKpW78pTkzyg6rQuiUD1zobf7iMiQUA0cPAUx9a2fT/Qwt1G9XOdyTnyVHWZe/sCXAXB96jqs+6Bbqnx8fE1/hJM7eZl5BLXPJQLu7dyOooxHpt6bgpTzunAC19vY+bibU7HaareBd5x//kpri+ObProJqZFeAiX9WrDG5n51q3NmHqWsf0gd76SSd/EFvznxgEEBXrS677xnCpNR1WdrKrP4FoMLBW4UlVXedh2OtDFPTtPCK4Bt9UXhlkITHI/vx74TF3Lbi4EJrhn8EnBdSt5eW1tuo9Z5G4Dd5tvnck5VHUPkCsi3dzHXAxs8PA9Gw/tO1rMZ5v2cd2gBIKb2F8KY+ry+yt6clmv1vz5nQ18tH6P03GaHFXto6p93X92wTV+62unc5nvG5eaSOGJMj7ZuNfpKMb4jC17jzJtVgYJLZoxffJgwkNqXXfXMae68vp2bWJVrQC2qepRTxt297e/A/gQ1yw681V1vYg8ICKj3bu9AMSKSDZwN+6uNqq6HpiP68L7A+CnqlpRW5vutu4B7na3Fetu+7TP4T7mZ7hmPFqDa8rTv3n6vo1n3liZT0WlMnaQzf1vvE9ggPCv8QPol9iCO+dmsirX5gk4FVVdCQx2Oof5vuGd4kho0Yz5GdYNyJj6sKewmEnTlxMSFMCsqWnERIQ4HalG4voyvIYXRCpw9X8HV3/5ZrhmAxJAVTWqURJ6idTUVM3IyHA6hldQVS5+9AtiwkNY8OPhTscx5oztP1bCtU8toai0nDd+cg5JMeFORzojIrJCVVPrsb27q/wYgKsbZayqXlbLIY3OPrP/59GPN/Ofz7aw+J6LaNeimdNxjPFahSfKGP/MN+QdOsHc24bSOyG6Qc5TH5/Ztd4BUNVAVY1yPyJVNajKc7v4N2dsxY5D5BQcZ5yt/Gu8XFzzUGZMGUx5pTJpxnIKi8rqPsg/RFZ5hOIaC+DJ2DHjgLGDElGF12wwsDFnrKS8gttfzGBrwTGevnlQg1381xfrfG0a3fyMXCJCArmiT1unoxhz1jrFN+fZiankHiziRy+toLS80ulITcEGVf2z+/Ggqs7BpgFtspJiwhneKZZXV+RRadPbGnPaKiuVu+evZmnOQR4e249zu8Q5HalOVgCYRnWspJx31uzmyr7tiAhteoNijDkTaSkx/PP6vnyTc4Dfv7mW2rpW+pH7PNxmmohxqUnsPFjEsm0HnY5ijFdRVf767kbeXbOb347qzpj+3rGwqV2BmUb17ppdFJVWWPcf43OuGZDItv1FPP7pFjrERfCTCzo7HanRicjlwCggQUQer/JSFFBe81GmKRjZuw2RbwUxPyOXYZ1inY5jjNd47qscpi/extRzUvjheR2djuMxuwNgGtX8jDw6xUcwMLmF01GMqXd3/aALo/u1458fZPHumt1Ox3HCLiADKAZWVHksBJrMAGDzfWHBgYzu14731u7mSLGNZTHGE29m5vO39zZxZd+2/P6KHk1qoa+61FkAiMhRETlS7ZErIm+IiPeUOsZx2fuOsmLHIcYPTvKqvyTGeEpE+Of1fUlt35K7569i5c5DTkdqVKq6WlVnAZ1VdVaVx+uq6l+/DC80fnASJeWVvL26+pqdxpjqvt6yn18vWM2wjrE8Mq4fAQHedV3jyR2AR4FfAwm4Vsv9FfAcMBeY3nDRjK+Zn5FHUIBwzYBEp6MY02DCggN5ZuIgWkeFcdvsDHIPFjkdyQlpIvKxiGwWkRwR2SYiOU6HMqfWJyGa7m0ibU0AY+qwLr+Q21/MoFN8c565ZRChQYFORzptnhQAI1X1GVU9qqpHVPVZYJSqzgNaNnA+4yPKKip5fWUeF3VvRXxkqNNxjGlQsc1DmT55MKXllUydmU7hCb/rUvECri+PzsW1AFgqthBYkycijE1NYnXuYbL2eLzupzF+JfdgEZNnpNMiPIRZU9OICgt2OtIZ8aQAqBSRcSIS4H6Mq/Ka3091YTzz2aZ97D9Wyngb/Gv8ROdWzXl64iC27T/OT+espKzCr6YHLVTV91V1n6oeOPk41QEiMlJEskQkW0TureH19iLyqYisEZHPRSSxymuTRGSL+zGpId6Qv7i6fzuCA4VXM3KdjmJMk3PweCm3TF9OWUUls6YOpnVUmNORzpgnBcBNwERgH7DX/fxmEWkG3NGA2YwPeTUjl1aRoYzoGu90FGMazfBOcfzt2j58nb2fP761zp+mB10kIv8nIsNEZODJR207i0gg8CRwOdATuEFEelbb7WFgtqr2BR4A/u4+Ngb4EzAESAP+JCJ2d/oMxTYP5Qc9WvNGZr6taWFMFUWl5Uydmc6uwyeYPjmVzq0inY50VuqcBlRVc6h9AZev6zeO8UX7jhSzKKuA287vSFCgTTxl/Mu41CS27z/OU59vJSUugtvO7+R0pMYwxP1n1aXqFbiolv3TgGz3vzeIyFxcKwdvqLJPT+Au9/NFwJvu55cBH6vqQfexHwMjgVfO8j34rXGpSby/bg+fbdrHyN5tnI5jjOPKKyr52cuZrMk7zNM3D2JQ+xinI521OgsAEYkHfgh0qLq/qk5tuFjGl7yemU9FpTJ2kA3+Nf7pV5d2Y8eBIv7+/iaSYyJ8/qJKVS88zUMSgKp9TvL4XxFx0mrgOuDfwDVApIjE1nJsjSvxiMhtwG0AycnJpxnRf5zXJY7WUaG8mpHr8/+vGlMXVeV3b6zj0037ePCa3lzayzf+TnjydexbQDTwCfBulYcxdVJV5mfkMrhDSzrGN3c6jjGOCAgQHhnXj/5JLfjFvExW5x52OlKDEpHWIvKCiLzv/rmniEw71SE1bKveX+pXwAgRyQRGAPm4Fhfz5FjXRtVnVTVVVVPj4607Ym2CAgO4bmAii7L2sfdIsdNxjHHUY59sYV5GLnde1JmbhrR3Ok698aQACFfVe1R1vqq+dvLR4MmMT1ix4xA5BccZl2qDf41/CwsO5LlbUolrHsqtszPIP3zC6UgNaSbwIdDO/fNm4Ben2D8PqPohkYhrUbFvqeouVb1WVQcAv3NvK/TkWHP6xqYmUanw2kqbEtT4rznLdvD4p1sYn5rEXZd0dTpOvfKkAHhHREY1eBLjk+Zn5BIREsioPm2djmKM4+KahzJj8mCKSyuYOiOdo7674mqcqs4HKgFUtRyoOMX+6UAXEUkRkRBgAq7Vg78lInEicvLfrPv43zo0HwKXikhL9+DfS93bzFlIiYsgLSWGVzPy/GnwujHf+mj9Hv7w5jou6t6KB6/p7XMLmHpSAPwcVxFwwr0K8FEROdLQwYz3O15SzjtrdnNl33ZEhNY53MQYv9CldSRP3TyQ7IJj3PFyJuW+OT3ocXf/fAUQkaFAYW07uwuEO3BduG8E5qvqehF5QERGu3e7AMgSkc1Aa+BB97EHgb/gKiLSgQdODgg2Z2d8ahLb9h9n+Tb7dRr/smLHQX72SiZ9ElvwxI0DfHICkzrfkapGqmqAqjZT1Sj3z1GNEc54t3fX7qaotIJxg23wrzFVndclnr9e3ZsvNhdw/9vrffEb1rtxfYPfSUQWA7OBn53qAFV9T1W7qmonVT15cf9HVV3ofr5AVbu497lVVUuqHDtdVTu7HzMa7m35l8v7tKF5aJCtDGz8Sva+Y0yblUG7Fs2YPimV8BDf/AKz1nclIt1VdVNtczer6sqGi2V8wasZuXSMj2Bgsk3JbUx1N6Qls33/cZ75MoeUuOZMOzfF6Uj1RlVXisgIoBuuQbpZquqz/Z18VXhIEFf1a8ebmfncP7onkV664qkxntp7pJhJ05cTFBDA7KlpxDYPdTpSgzlVWXM3runSHqnhtVPN52wMOQXHSN9+iHsv7+5z/eaMqS/3jOzOjgNF/PXdDSTHhHNJz9ZOR6oX7oW9RvG/6aMvFRFU9VFHg5nTNn5wEq8s38nbq3dz4xCbOtX4riPFZUyavpzDRaXMu30YSTHhTkdqULV2AVLV29x/XljDwy7+zSm9uiKPwADh2gE1TsdtjME1Pehj4/vTJyGaO1/JZF1+rd3kvc3bwGQgFois8jBepl9iNN1aRzIvI7funY3xUiXlFfzoxRVk7zvG0xMH0Tsh2ulIDc6jjk0iMpzvLwQ2u4EyGS9XXlHJayvyuLBbPK2iwpyOY0yT1iwkkOdvSeXqJxczbVY6b/70HNpGN3M61tlKVNW+TocwZ09EGJuayF/f3UjWnqN0a2N1nPEtlZXK3fNXs2TrAR4b34/zuvjHGiF1DgIWkReBh4FzgcHuR+opDzJ+7cstBew7WsJYm/vfGI+0igpj+pTBHC+pYOrMDI6VlDsd6Wy9LyKXOh3C1I9rByYSHCjMt7sAxseoKg+8s4F31+zmt6O6c80A/5m0xJN5jVKBc1T1J6r6M/fjzoYOZrzX/PQ84pqHcFH3Vk5HMcZrdG8TxRM3DmDz3qPc+YrXTw+6FHjDpo/2DTERIVzSszVvZOZTWu7V/18a8x1Pf5HDzCXbufXcFG47v5PTcRqVJwXAOqBNQwcxvuHAsRI+2biXawYkEOyD8+Ya05Au6NaK+0f34rNN+/jLOxu8eXrQR4BhuFaSt+mjfcDY1CQOHi/l0417nY5iTL1YsCKPhz7YxOh+7fjtqB5Ox2l0nowBiAM2iMhyoOq8y6NrP8T4qzcy8ymvVOv+Y8wZmji0PTv2H+f5r7eRHBvhrdODbgHWqRdXMOa7zu8ST9voMOZl5HK5rexuvNyirH3c89oazu0cx8Nj+xEQ4H+zFXpSANzf0CGMb1BV5mfk0j+pBV1b20AxY87Ub0f1IO/QCf767gYSWzbjsl5edxN2N/C5iLzPd784smlAvVRggHD9oESeXJTN7sITvjBQ3fipVbmH+clLK+nRNpKnJw4iJMg/eyuc8l2753L+g6p+Uf3RSPmMF1mTV8jmvccYZ9/+G3NWTk4P2jexBT+fm8nq3MNORzpd24BPgRBsGlCfMXZQEpUKC2xlYOOlcgqOMXVmOvGRocyYnEbzUN9c5dcTpywAVLUCKBIR358Q1Zy1eRm5hAUHcGU/uz1szNk6OT1oXPNQps1KJ/dgkdORPKaqf1bVPwOPAo9U+dl4seTYcIZ1jOXVFXlUVlrvLuNd9h0p5pbpyxFg9tQ04iN9d5VfT3hy36MYWCsiL4jI4ycfDR3MeJcTpRW8vWoXo3q3JcqWizemXsRHhjJzymBKyyuZMjOdwhNlTkfyiIj0FpFMXJNIrBeRFSLSy+lc5uyNH5zEzoNFLN12wOkoxnjsSHEZk2akc/B4KTOmDKZDXITTkRznSQHwLvAH4EtgRZWHMd/6YP1ujpaU2+BfY+pZ51aufqo7Dhznxy+t8JZpGJ8F7lbV9qraHvgl8JzDmUw9GNm7DZFhQcxPtzUBjHcoKa/g9tkr2LL3KE/fPIi+iS2cjtQk1FkAqOqsmh6NEc54j/npeSTHhDMkJcbpKMb4nOGd4vjHtX1ZsvUAv31jrTdMDxqhqotO/qCqnwP2lZsPCAsOZEz/dry/bo/X3JEy/uvkKr/f5Bzg/8b25fyu/rHKryc8WQm4i4gsEJENIpJz8tEY4Yx32L7/ON/kHGBcaqJfTqVlTGO4blAiP7+4CwtW5PHEZ9lOx6lLjoj8QUQ6uB+/xzUw2PiA8anJlJRXsnD1LqejGFMrf17l1xOedAGaAfwXKAcuBGYDLzZkKONd5qbnEhgg1v3HmAb2ix904doBCTzy8WbezMx3Os6pTAXigdeBN9zPpziayNSb3glR9GgbZd2ATJP23y+2+u0qv57wpABopqqfAqKqO1T1fuAiTxoXkZEikiUi2SJybw2vh4rIPPfry0SkQ5XX7nNvzxKRy+pqU0RS3G1scbcZcqbncL8WKCKZIvKOJ+/VX5WWV7JgRS4XdW9F66gwp+MY49NEhL9f14chKTH8ZsEaluU0zYGYqnpIVe9U1YGqOkBVf66qh5zOZeqHiDA+NZG1+YVs2HXE6TjGfM/c5Tv55wdZfrvKryc8mgVIRAKALSJyh4hcA7Sq6yD3GgJPApcDPYEbRKRntd2mAYdUtTPwGPCQ+9iewASgFzASeMp9QX6qNh8CHlPVLsAhd9unfY4q2X4ObPTg9+PXPt24l/3HSrkhzb79N6YxhAYF8uzEVJJimnHbiyvYWnDM6UjfEpGFp3o4nc/UnzH9EwgJDGB+ht0FME3L+2t389s31jKia7zfrvLrCU8KgF8A4cCdwCDgZmCSB8elAdmqmqOqpcBcYEy1fcYAJwcULwAuFhFxb5+rqiWqug3IdrdXY5vuYy5yt4G7zavP8ByISCJwBfC8B+/Tr72Snkvb6DBGdK2zJjTG1JPo8GBmTE4jKECYMiOdA8dK6j6ocQwDEoGvgIeBR6o9jI9oGRHCpb1a8+aqfErKK5yOYwwAX2/Zz8/nrmJAckv+e/NAv13l1xOezAKUrqrHcH2LPkVVr1PVpR60nQBU/Wogz72txn1UtRwoBGJPcWxt22OBw+42qp/rdM8B8C/gN8Ap59sTkdtEJENEMgoKCk61q0/KPVjEV1sKGJeaRKBV2MY0quTYcJ6flMreI8X8cHYGxWVN4iKsDfBboDfwb+ASYL+tIO+bxg9O4nBRGR+t3+t0FGNYlXuY217MICUugumTBhMe4r+r/HrCk1mAhonIBtzdYUSkn4g85UHbNV0RVp+7rrZ96mv7aZ9DRK4E9qlqnWsdqOqzqpqqqqnx8f43tdTJW7/jBlv3H2OcMCC5Jf8a35/M3MP8cv5qx1dnVdUKVf1AVScBQ3HdWf1cRH7maDDTIM7pFEdCi2bMTd/pdBTj57L3HWXyjOXENg/hxWlpRIfbgqR18eTeyL+Ay4ADAKq6Gjjfg+PygKpXholA9TnDvt1HRIKAaODgKY6tbft+oIW7jernOt1znAOMFpHtuLoYXSQiL3nwfv1KeUUl8zNyuaBrPAktmjkdxxi/dXmfttx3eXfeXbubf36Y5XSckxMvXAu8BPwUeBzXbEDGxwQECDekJbE4+wDb9h93Oo7xU3mHirj5+eUEBwbw0rQhtLIJSTziUecoVa0+yseTe83pQBf37DwhuAbcVh8EtpD/jSe4HvhMXSvcLAQmuP8hSQG6AMtra9N9zCJ3G7jbfOtMzqGq96lqoqp2cLf/mare7MH79SuLsgrYe6SECWnJTkcxxu/98LyO3Dw0mYztBx3tjy0is4AlwEDgz6o6WFX/oqpNes5Sc+bGpSYRFCC8stzuApjGt/9YCbe8sJzjpeXMnppG+1hbb9BTnnSQyhWR4bi6x4TgGgxc5+w4qlouIncAHwKBwHRVXS8iDwAZqroQeAF4UUSycX0rP8F97HoRmQ9swLX+wE9VtQKgpjbdp7wHmCsifwUy3W1zJucwdZu7fCfxkaFc1N0G/xrjNBHh/qt6UV6phAYF1n1Aw5kIHAe6Ane65lsAXF0uVVWjnApmGkarqDAu6dmaVzNy+eWlXZ3+/8/4kaPFZUyesZxdhSd4adoQerS1j5fTIXUtKS8icbgGc/0A14f4R8Cdqnqw4eN5j9TUVM3IyHA6RqPYXXiCc/7xGT++oBO/vqy703GMMWdJRFaoaqrTORqTP31mN7SvthQw8YXl/HtCf8b0rz7XhzH1r7isgknTl7NixyGem5TKhd3868vI+vjM9mQWoP2qepOqtlbVVu7uMLeczUmNd5ufnkelupaDN8YY49/O6RRHckw4c5ZZNyDT8MorKrnj5UyWbz/II+P6+d3Ff3050wlS767XFMZrVFQq8zNyOa9LHMmx4U7HMcYY4zDXYOBklm87SPa+o07HMT6solL55aur+WTjXu6/qpfdcToLZ1oA2KTvfurLzQXkHz7BhMH27b8xxhiXsamJBAeK3QUwDUZV+f2ba3lr1S5+fVk3Jg3v4HQkr3amBYCzk00bx7y0dAdxzUO5pGdrp6MYY4xpIuKah3JZrza8tiKvqSxKZ3yIqvLAOxt4ZXkud1zYmZ9e2NnpSF6v1gJARI6KyJEaHkeBdo2Y0TQRuQeL+CxrHzekJdny2sYYY77jxiHJHCku5901u52OYnzMwx9lMWPxdqaek8IvL+3qdByfUOtVnKpGqmpUDY9IVbX1lf3Qy8t3EiDCjUOs+48xpn6JyEgRyRKRbBG5t4bXk0VkkYhkisgaERnl3h4sIrNEZK2IbBSR+xo/vQEY1jGWjnERzFm2w+koxoc8uSibJxdt5Ya0JP5wZQ+qTC9szoJ9jWs8UlJewbz0XH7QoxVto23lX2NM/RGRQOBJ4HKgJ3CDiPSsttvvgfmqOgDXei5PubePBUJVtQ8wCLhdRDo0Rm7zXeL+gmjlzsNs2nPE6TjGB0z/ehv/92EWV/dvx1+v7mMX//XICgDjkffX7uHg8VImDu3gdBRjjO9JA7JVNUdVS4G5wJhq+yhwcqWfaGBXle0RIhIENANKAbv6dMh1AxMJCQrgZRsMbM7S3OU7eeCdDYzs1YaHx/YjMMAu/uuTFQDGI7O/2U7HuAiGd4p1OooxxvckALlVfs5zb6vqfuBmEckD3gN+5t6+ANfqw7uBncDDtlClc1pGhHBFn7a8vjKfYyXlTscxXurNzHzue2MtF3SL5/EbBhAUaJer9c1+o6ZO6/ILWbnzMDcNbU+AVeDGmPpX0wdL9dnmbgBmqmoiMAp4UUQCcN09qMA1OUUK8EsR6VjjSURuE5EMEckoKCiov/TmO24Z1p5jJeW8tiLP6SjGC723dje/fHU1Q1NiefrmQTbpSAOx36qp05xlOwgLDuD6gYlORzHG+KY8IKnKz4n8r4vPSdOA+QCq+g0QBsQBNwIfqGqZqu4DFgOpNZ1EVZ9V1VRVTY2Pj6/nt2BOGpDckn6J0cz6ZjuVlTZruPHc+2t387NXMhmQ1ILnJ6USFhzodCSfZQWAOaXCE2W8mbmLMf0SiA4PdjqOMcY3pQNdRCRFREJwDfJdWG2fncDFACLSA1cBUODefpG4RABDgU2NltzUaPI5HcgpOM7X2fudjmK8xAfrXBf//ZNaMHNqGhGhNuFkQ7ICwJzS6yvzOFFWwcRh7Z2OYozxUapaDtwBfAhsxDXbz3oReUBERrt3+yXwQxFZDbwCTFZVxTV7UHNgHa5CYoaqrmn0N2G+Y1SftsQ1D2HWku1ORzFe4IN1e7jj5Uz6JkYzc8pgmtvFf4Oz37CpVWWlMvubHfRLakHvhGin4xhjfJiqvodrcG/VbX+s8nwDcE4Nxx3DNRWoaUJCgwK5MS2Z/yzKZueBIpJjw52OZJqoj9bv4Y6XV9InMZpZU9OIDLPeBo3B7gCYWn2+eR/b9h9n6jkdnI5ijDHGy9w0tD2BIsz+ZrvTUUwT9fGGvfz05ZX0TrCL/8ZmBYCp1fSvt9M6KpRRfdo6HcUYY4yXaR0VxsjebZiXkctxmxLUVPPJhr38ZM4KerWLZva0NKLs4r9RWQFgapS15yhfZ+/nlmEdCLb5d40xxpyBKed04GhxOW9k5jsdxTQhn2zYy4/nrKCnXfw7xq7sTI1mLN5GWHAAN6YlOx3FGGOMlxqY3JLeCVHM/mY7rjHbxt+9u2Y3P3ppBT3bRjF7ql38O8UKAPM9B46V8HpmPtcOTKRlRIjTcYwxxngpEWHSsA5s3nuMxdkHnI5jHPZGZh4/e2UlA5Jb8NKtQ4huZhf/TrECwHzPK8t3UlpeyZThHZyOYowxxsuN7t+OuOahPPdVjtNRjINeXraTu+evZmjHWBvw2wRYAWC+o7S8ktnf7OD8rvF0aR3pdBxjjDFeLjQokCnndOCLzQVs2nPE6TjGAdO/3sZv31jLBV3jmT55MOEhNgu906wAMN/x3trd7DtaYlN/GmOMqTc3DUmmWXAgz325zekoppE99Xk2D7yzgZG92vDMxFTCggOdjmSwAsBUoao882UOneIjOL9LvNNxjDHG+IgW4SGMH5zEwtX57CksdjqOaQSqyqMfZfHPD7IY078dT9w4gJAgu+xsKuy/hPnWl1v2s3H3EW4f0YmAAHE6jjHGGB8y7dwUKiqVmUu2Ox3FNLDKSuXPb2/g8c+yGZeayKPj+hNkU4o3KfZfw3zrv59n0yYqjKv7JzgdxRhjjI9Jignn8j5tmbNsB8dsYTCfVVpeyV3zVzFzyXamnpPCP67tS6B9qdjkWAFgAFiVe5ilOQe59bwUu0X3/+3dd3hVVb7/8fc3CUkg9NBCB6kBRYoIoojoUNQR24xgV0YcUa9lRsXRe++M4/zUe+33CuqIYxccdIQfojhKUSx0pCOhhyKhhV6SrPvHWTjHmIDBk+xTPq/nOU/2Xnvttb/rrGRlr11FRKRcDDurJXsOFjBm1vqgQ5FysP9wATe9NofxCzZx74C2/PuF7XVFQZTSnp4A8Py0VVRPT2GwWlJsNAAAGjRJREFUXvwlIiLlpFOTmnRvUZuXZ6zhSGFR0OFIBO3af5irX5rJ5yvzePTSkxnepxVm2vmPVhoACKvy9jJ56Rau7dmcqml6NJeIiJSf357dkk35B3l//sagQ5EI2ZJ/kF+/8BWLN+5m5FVddDAxBmgAILw4fTWpyUlcr0d/iohIOTunbT2ys6ozctoqCotc0OHIz7Qqby+XjfqSTbsO8sqNpzGgY1bQIclPoAFAgtuSf5D35ufy625NqFM1LehwREQkzpkZt/dtxZpt+5i4cFPQ4cjPMGvNDi4b9SUHjxQyZlgPzjipTtAhyU+kAUCCGzUtB+dgWO+WQYciIiIJon+HBrSpX5XnpuZQpLMAMWnCN5u4+qWZ1M5I5R/De9GxUY2gQ5Iy0AAggW3JP8jbszZwedfGNKldJehwREQkQSQlGbee04pvv9vL5CVbgg5HysA5x8hpOfzb2/M5tWlN3rvlDJpmah8i1pTrAMDMBpjZCjPLMbMRJSxPM7OxfvlMM2setux+n77CzPofr0wza+HLWOnLTD2RbZhZEzObambLzGyJmd1RHt9NNBg1LYci57j1nFZBhyIiIgnmwlMa0qJOBv8zJQfndBYgFhQUFvGHfyzmvz5awUWdGvL60O7UrJIadFhyAsptAGBmycBzwEAgGxhiZtnFsg0FdjrnWgFPAY/5dbOBwUAHYAAw0sySj1PmY8BTzrnWwE5fdpm3ARQAv3POtQd6ALeWEHfM09F/EREJUnKSMbzPSSzdvJtPl20NOhw5jr2HChj66hzenrWeW885iaevOJW0lOSgw5ITVJ5nALoDOc651c65w8AYYFCxPIOAV/30OOBcCz00dhAwxjl3yDm3Bsjx5ZVYpl+nry8DX+bFJ7IN59xm59w8AOfcHmAZEHevxtXRfxERCdrFnRvRpHZlnvrkW90LEMXWb9/PpSO/YEbONh659GTu6d9OL/iKceU5AGgEbAibz+XHO9Lf53HOFQD5QOYx1i0tPRPY5csovq2ybuN7/nKhzsDMkipoZsPMbI6ZzcnLyyspS1TS0X8REYkGlZKTuOu8NizZtJtJizcHHY6U4MtV27jouRl8t/sQr93YnSF6xn9cKM8BQElDw+LD+9LyRCr9RLYRWsmsKvAucKdzbncJeXHOveic6+ac61a3bt2SskSlZ6es1NF/ERGJCoNObUSb+lV58uNvKdDbgaOGc47Xv1rLNaNnUbdqGhNu60WvVnrMZ7wozwFALtAkbL4xUPyBv9/nMbMUoAaw4xjrlpa+Dajpyyi+rbJuAzOrRGjn/03n3HtlqHPUW5W3l7GzN3DV6U119F9ERAKXnGT8vl9bVm/bx7i5uUGHI8DhgiIeeH8x/z5+CX3a1OW94WfQLDMj6LAkgspzADAbaO2fzpNK6IbbCcXyTACu89OXA1Nc6FEAE4DB/gk+LYDWwKzSyvTrTPVl4MscfyLb8PcHjAaWOeeejNi3ESUen7yC9JQkbj+3ddChiIiIAPCL7Pp0blqTZz5dycEjhUGHk9C27jnI1aNn8tbM9QzvcxIvXtuNaumVgg5LIqzcBgD+evvbgMmEbqR9xzm3xMweMrOLfLbRQKaZ5QB3AyP8ukuAd4ClwEfArc65wtLK9GXdB9zty8r0ZZd5G0Av4Bqgr5kt8J/zy+ErqnDz1u/kw8VbGNb7JL31V0REooaZcU//tmzOP8gbX68LOpyENWvNDi58dgYLc3fxzOBTuXdAO5J1s29cMj17NzK6devm5syZE3QYpXLOccWLX7M6bx/T7+lDRlrK8VcSkYRgZnOdc92CjqMiRXufnaiuGT2TRRvzmfb7Pnq+fAVyzvHS52t49KPlNK1dhVFXd6Fdg+pBhyWliESfrTcBJ4hPl21l1pod3HFea+38i4hIVPrD+e3ZfeAIT3+yMuhQEsbug0e45Y15/GXSMn7Rvj4Tbuulnf8EoAFAAjhUUMjDHyylZd0MBp/W5PgriIiIBKB9VnUGd2/K61+vI2frnqDDiXtLNuVz0f/M4J/LvuPBC9oz6uouut4/QWgAkABGz1jD2u37+eMvO1ApWU0uIiLR63e/aEOV1GT+PHFZ0KHEraIix+gZa7jkuS/Zf7iQMcN68JuzWhJ6DookAu0Nxrkt+Qf53yk59MuuT+82sfOuAhERSUyZVdO449zWTP82j6nLtwYdTtzJ23OIG16ZzZ8nLqV3m7p8dGdvTmteO+iwpIJpABDnHvlwGQVFjgcvyA46FBGRUpnZADNbYWY5ZjaihOVNzWyqmc03s4XhT2czs1PM7CszW2Jmi8wsvWKjl0i7tmdzWtbJ4KGJS/VY0AiaumIrA5/5jK9Xb+fPF3fkr9d2pXaGbrZORBoAxLGvVm1n/IJN3Ny7JU0z9dIvEYlOZpYMPAcMBLKBIWZW/KjFg4Qe/dyZ0DtgRvp1U4A3gN865zoAfYAjFRS6lJPUlCT+NKgDa7btY+TUnKDDiXn7DxfwxwlLuOFvs6lTNY3/f/uZXNOjmS75SWB6HEycOnikkPvfW0jT2lUY3qdV0OGIiBxLdyDHObcawMzGAIMIvaflKAccfTRJDf71tvd+wELn3DcAzrntFRKxlLuzWtflks6NGDV9Fb/s1JDW9asFHVJMmrl6O/e+u5B12/dzQ6/m3DegHemVkoMOSwKmMwBx6ulPVrJ2+34eufRkKqfqD11EolojYEPYfK5PC/dH4GozywUmAbf79DaAM7PJZjbPzO4tbSNmNszM5pjZnLy8vMhFL+XmwQvak5GWwv3vLaKoSO8tKoujR/2vePFrnIMxw3rwn7/soJ1/ATQAiEuLN+bz189X8+tujenVqk7Q4YiIHE9J1yEU39sbArzinGsMnA+8bmZJhM5knwlc5X9eYmbnlrQR59yLzrluzrludevqoQixILNqGg+c354563by1qz1QYcTM75evZ2Bz3zOK1+u5fozmvPRnWfRo2Vm0GFJFNElQHHmcEER945bSO2MVB44Xzf+ikhMyAXCX1LSmH9d4nPUUGAAgHPuK3+jbx2/7nTn3DYAM5sEdAE+Le+gpWJc3rUx4xds4v9NWsaZrerQvE5G0CFFre17D/HIh8sZNzeXJrUr8/ZNPeh5knb85cd0BiDOPPHxCpZu3s1fLu5IjSp6mYeIxITZQGsza2FmqYRu8p1QLM964FwAM2sPpAN5wGTgFDOr4m8IPpsf3jsgMc7M+O9fnUJKknHXOwsoKCwKOqSoU1TkGDNrPX2fmM778zdyS5+T+PjOs7XzL6XSACCOfJGzjRc+W82VpzelX4cGQYcjIvKTOOcKgNsI7cwvI/S0nyVm9pCZXeSz/Q64ycy+Ad4GrnchO4EnCQ0iFgDznHMfVHwtpDxl1ajMw5eczPz1uxg5bVXQ4USVJZvyufz5Lxnx3iLaNqjGpDvO4r4B7XT/nxyTLgGKEzv3HebudxbQsm4GD17QPuhwRETKxDk3idDNveFp/xE2vRToVcq6bxB6FKjEsYs6NeTTZd/xzKcr6dUqk67NEvvlVVt3H+Txj1fw97m51KqSyuO/6sRlXRrp0Z7yk2gAEAcKixx3jF3Ajn2HGX3daVRJVbOKiEj8eWhQRxZs2MXwN+cx8fazqFstLeiQKtyBw4X89fPVPD99FUcKi/jNmS247ZzWuuxXykSXAMWBJz5ewWff5vHQoI50bFQj6HBERETKRY3KlRh1VVfyDxzhtrfmJdT9AEcKixg7ez19n5jGk//8lrPb1OWTu8/mgQuytfMvZaYBQIz7cNFmRk5bxZDuTRnSvWnQ4YiIiJSr7IbVeeTSk5m5Zgd/mbQs6HDKXUFhEePm5nLuE9O5791F1Kuezjs392TU1V1plqknIsmJ0bUiMWzuuh3cOXYBnZvW5I8X6ZGfIiKSGC7p3JiFufn87Yu1NK5VhaFntgg6pIgrKCxi4sLNPPvpSlZv20fHRtV5+fpunNO2nq7zl59NA4AYlbN1L0NfnUNWjXReurYbaSm6219ERBLHgxdks2nXAR7+YCkNqqdzwSlZQYcUEXsPFTB29gZenrGGjbsO0K5BNV64piv9sutrx18iRgOAGLR++36ue3kWKUlJvHbj6WRWTbyboEREJLElJxnPDO7M1S/N5K6xC6icmkTfdvWDDuuEbck/yKtfreXNr9ex+2AB3ZvX5k8XdaBvu3okJWnHXyJLA4AYszpvL1f+dSYHCwp5Y+jpNM2sEnRIIiIigUivlMxL13XjmtGzuPn1uTx3ZZeYeg9OYZHjs5V5vDVzPVOWb8U5x4CODbjprJZ0blor6PAkjmkAEEMW5eZz46uzQ2/8G9aDdg2qBx2SiIhIoGpWSeWN35zOtS/PYvib83j0slO4vGvjoMM6pg079jN+wUbGzN5A7s4D1KmayrDeLRlyWlMd2JMKoQFAjJi0aDN3v7OAzIw0Xr3pNFrVqxZ0SCIiIlGhRuVKvDG0Oze/Ppff//0bcrbu5d7+baPq0pm8PYf4YOEmxn+zifnrdwFwxkmZjBjYjn7ZDUhN0YMZpeJoABDlDh4p5NEPl/PKl2vp2qwWL1zTlTq65l9EROQHqqVX4tUbu/OfE5bw/PRVLNq4i8d/1YmsGpUDicc5x7ff7WXK8q1MWf4dc9ftpMhB+6zq3DegHb/slEXjWjraL8HQACCKfZmzjf+YsIScrXu5/ozmjBjYjvRKetqPiIhISSolJ/GXiztySqMaPDRxKf2f+owRA9tzxWlNSK6AswGbdh1g1podzFyzg89X5pG78wAAHRpW57ZzWnFhp4a0qa8z+BI8DQCijHOOeet3MmraKj5ZtpVGNSvz2o3d6d2mbtChiYiIRD0zY3D3pvQ8KZN7xi3kD/9YxGtfreX2vq3p36E+KcmRudQmf/8RlmzOZ+mm3SzZtJs563awYUdoh79aWgqnt8xkeJ9W9G1XjwY10iOyTZFI0QAgChw8UsiKLXuYkbONSYs2s2TTbqqnp3BP/7YMPbOFjvqLiIiUUbPMDMYO68FHi7fw2EfLufWteWTVSOf8k7M4p209OjaqTs0qqSWu65xj94ECtu07xI59h8nbc4h12/ezfsc+1u/Yz9pt+9m468D3+etVS6Nz05rccEYLureoTfus6hVyxkHkRGkAEJApy7/j4Q+WsftAATv3H6awyAFwapOaPDSoA5d1aUxGmppHRETkRJkZA0/Ool+HBkxdvpU3Z67j9a/WMXrGGgBqValE1fQUUpKSOFxQxKGCQg4VFHHgcCEF/v9yuNoZqTStXYVuzWtxdYNmZDesTnZWdepW0715Elu0hxmQGpVTaZ9VnerplahbLY3srGp0blqL+tV1mlBERCSSkpOM87Lrc152ffYdKmD22h2s2LKH9Tv2s/9wIUcKi0hLSSatUhJpKUlUrpRMZtU0MjNSqZ2RSp2qaTSpXZlq6ZWCropIRGgAEJCuzWrRtZle8iEiIlKRMtJS6NO2Hn3a1gs6FJHA6KGzIiIiIiIJRAMAEREREZEEogGAiIiIiEgCKdcBgJkNMLMVZpZjZiNKWJ5mZmP98plm1jxs2f0+fYWZ9T9emWbWwpex0peZGultiIiIiIjEunIbAJhZMvAcMBDIBoaYWXaxbEOBnc65VsBTwGN+3WxgMNABGACMNLPk45T5GPCUc641sNOXHeltiIiIiIjEtPI8A9AdyHHOrXbOHQbGAIOK5RkEvOqnxwHnmpn59DHOuUPOuTVAji+vxDL9On19GfgyL47kNiL0nYiIiIiIBKo8BwCNgA1h87k+rcQ8zrkCIB/IPMa6paVnArt8GcW3Falt/IiZDTOzOWY2Jy8vr6QsIiIiIiJRpTwHACW9A7v4a/VKyxOp9Ehu48eJzr3onOvmnOtWt27dkrKIiIiIiESV8nwRWC7QJGy+MbCplDy5ZpYC1AB2HGfdktK3ATXNLMUf5Q/PH6ltHNPcuXO3mdm64+Urpo6PPZ6pjvFBdYwPpdWxWUUHErQT7LMhsX9P4onqGB8StY4/u88uzwHAbKC1mbUANhK64fbKYnkmANcBXwGXA1Occ87MJgBvmdmTQEOgNTCL0NH5H5Xp15nqyxjjyxwfyW0cr7LOuTKfAjCzOc65bmVdL5aojvFBdYwPiVDHn+pE+mxIjO9QdYwPqmN8KK86ltsAwDlXYGa3AZOBZOBl59wSM3sImOOcmwCMBl43sxxCR+UH+3WXmNk7wFKgALjVOVcIUFKZfpP3AWPM7GFgvi+bCG9DRERERCSmmXMlXt4uFUAj1/igOsYH1VF+ikT4DlXH+KA6xofyqqPeBBysF4MOoAKojvFBdYwPiVDH8pYI36HqGB9Ux/hQLnXUGQARERERkQSiMwAiIiIiIglEAwARERERkQSiAUBAzGyAma0wsxwzGxF0PGVhZk3MbKqZLTOzJWZ2h0+vbWb/NLOV/mctn25m9qyv60Iz6xJW1nU+/0ozuy6oOpXEzJLNbL6ZTfTzLcxspo91rJml+vQ0P5/jlzcPK+N+n77CzPoHU5PSmVlNMxtnZst9e/aMw3a8y/+eLjazt80sPdbb0sxeNrOtZrY4LC1i7WZmXc1skV/nWTMr6QWJCUV99vdlRe3fOsR/v60+OzbbMSr7bOecPhX8IfR40VVASyAV+AbIDjquMsSfBXTx09WAb4Fs4L+AET59BPCYnz4f+JDQOxZ6ADN9em1gtf9Zy0/XCrp+YfW8G3gLmOjn3wEG++nngVv89HDgeT89GBjrp7N926YBLXybJwddr2J1fBX4jZ9OBWrGUzsCjYA1QOWwNrw+1tsS6A10ARaHpUWs3Qi9E6WnX+dDYGDQbRnw75H67OP8zkTLhzjvt1GfHZPtSBT22YE3diJ+fCNNDpu/H7g/6Lh+Rn3GA78AVgBZPi0LWOGnXwCGhOVf4ZcPAV4IS/9BvoDr1Bj4FOgLTPR/VNuAlOJtSOidET39dIrPZ8XbNTxfNHyA6r6jtWLp8dSOjYANvsNM8W3ZPx7aEmhe7J9JRNrNL1selv6DfIn4UZ8d/X/rPp647rfVZ8d2O0Zbn61LgIJx9Bf8qFyfFnP86bbOwEygvnNuM4D/Wc9nK62+0fw9PA3cCxT5+Uxgl3OuwM+Hx/p9PfzyfJ8/musHoaOZecDf/Cnzl8wsgzhqR+fcRuBxYD2wmVDbzCX+2hIi126N/HTx9EQWC+3/k8Rxnw3x32+rzw6J9XY8KtA+WwOAYJR0bZar8Ch+JjOrCrwL3Omc232srCWkuWOkB8rMLgS2OufmhieXkNUdZ1lU1i9MCqFTkqOcc52BfYROQ5Ym5urpr6kcROgUcEMgAxhYQtZYb8tjKWudYrmu5SUuvpN47bMhYfpt9dn/EsvteDwV0mdrABCMXKBJ2HxjYFNAsZwQM6tE6B/Jm86593zyd2aW5ZdnAVt9emn1jdbvoRdwkZmtBcYQOp38NFDTzFJ8nvBYv6+HX14D2EH01u+oXCDXOTfTz48j9M8lXtoR4DxgjXMuzzl3BHgPOIP4a0uIXLvl+uni6YksFtr/mOK8z4bE6LfVZ4fEejseFWifrQFAMGYDrf1d7amEblyZEHBMP5m/u3w0sMw592TYognAdX76OkLXmR5Nv9bf2d4DyPenuyYD/cyslh/19/NpgXLO3e+ca+yca06obaY4564CpgKX+2zF63e03pf7/M6nD/ZPKWgBtCZ0o05UcM5tATaYWVufdC6wlDhpR2890MPMqvjf26N1jKu29CLSbn7ZHjPr4b+za8PKSlTqs6P8bz0R+m312fHRjmGC7bODvCEikT+E7vL+ltCd6Q8EHU8ZYz+T0OmlhcAC/zmf0HV3nwIr/c/aPr8Bz/m6LgK6hZV1I5DjPzcEXbcS6tqHfz1NoiWhDiQH+DuQ5tPT/XyOX94ybP0HfL1XEIVPUgFOBeb4tnyf0JMF4qodgT8By4HFwOuEngoR020JvE3o+tgjhI7+DI1kuwHd/Pe1Cvhfit10mIgf9dnH/p2Jpk8899vqs2OzHaOxzza/ooiIiIiIJABdAiQiIiIikkA0ABARERERSSAaAIiIiIiIJBANAEREREREEogGACIiIiIiCUQDAJEIMbO9/mdzM7sywmX/odj8l5EsX0Qk0ajPlkSmAYBI5DUHyvTPxMySj5PlB/9MnHNnlDEmEREpWXPUZ0uC0QBAJPIeBc4yswVmdpeZJZvZf5vZbDNbaGY3A5hZHzObamZvEXrZB2b2vpnNNbMlZjbMpz0KVPblvenTjh65Ml/2YjNbZGZXhJU9zczGmdlyM3vTvyEQM3vUzJb6WB6v8G9HRCS6qM+WhJMSdAAicWgE8Hvn3IUA/p9CvnPuNDNLA74ws4993u5AR+fcGj9/o3Nuh5lVBmab2bvOuRFmdptz7tQStnUpoTdDdgLq+HU+88s6Ax2ATcAXQC8zWwpcArRzzjkzqxnx2ouIxBb12ZJwdAZApPz1A641swXATEKv/27tl80K+0cC8G9m9g3wNdAkLF9pzgTeds4VOue+A6YDp4WVneucKwIWEDrNvRs4CLxkZpcC+3927URE4ov6bIl7GgCIlD8DbnfOneo/LZxzR48m7fs+k1kf4Dygp3OuEzAfSP8JZZfmUNh0IZDinCsgdATrXeBi4KMy1UREJP6pz5a4pwGASOTtAaqFzU8GbjGzSgBm1sbMMkpYrwaw0zm338zaAT3Clh05un4xnwFX+GtW6wK9gVmlBWZmVYEazrlJwJ2ETkWLiCQy9dmScHQPgEjkLQQK/GnhV4BnCJ3Knedv6sojdCSnuI+A35rZQmAFoVPKR70ILDSzec65q8LS/wH0BL4BHHCvc26L/2dUkmrAeDNLJ3Qk6q4Tq6KISNxQny0Jx5xzQccgIiIiIiIVRJcAiYiIiIgkEA0AREREREQSiAYAIiIiIiIJRAMAEREREZEEogGAiIiIiEgC0QBARERERCSBaAAgIiIiIpJA/g8YDZXoVr5IMwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot_lr(show_moms=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:21:06.711105Z",
     "start_time": "2019-08-21T02:21:05.287164Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEGCAYAAABcolNbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZwUxf3/8ddnd4HlvpYbEVAQERFwRRBBEFBAI5gQlXigxh9Bk3jF6CpqvLOJxivmG4NEPOMdxQiIsqLiAbggAiLKKQLLfd971O+P7h1md2f2bmaP9/PxmMd0V1f1VM3szme6u7rKnHOIiIgEJS7WFRARkapNgUZERAKlQCMiIoFSoBERkUAp0IiISKASYl2BoykpKcm1b98+1tUQEalU5s+fv9U516y05atVoGnfvj3p6emxroaISKViZj+WpbxOnYmISKAUaEREJFAKNCIiEigFGhERCZQCjYiIBEqBRkREAqVAIyIigapW99FUKM7Bod3gcrxll1PEo4R5crIgJxtctvccWs5Nz/HTs8LyZEVJ98s6B2ZgcYD5y+YvxxW+HLFM7vbcR/yR5bho6fH5yuXfFvZ6OTlH2uDCl7OPbMttb558+dNy8+cALl+7wtuWv61R2h5pe1yC94ivAXE1/Od869G25b4nIhWUAk2sZB6A1HaxroVUFXHhgSjhyLpzgIvyTCHbnBdTI5Yh748FDIx860U9xxWjbFzk8qFAHaFceFDPfS70PSiqvYW9d+WttHUqKg8wbhY0PS6gehdNgSZW4mvCuQ/l+0Wf/xd+pEdRefx/sLgE75duXIL3qz/O//Wfm27x/vb4Yqbn7puwI6fcP+Swo6ncP/ACyxReJvxoIvyRJz0731Fb/m3+PnPTQ20IO/IJb2Oeo6D4fPnzp/n5coXXI+J7EL5O4dtDR5GZkJ3pP2eFrWdB9uFCtmVGWc8q+sscShAc8h015fk8S/pM3r+RaM8R8xT12vm2lah9Ye0sKl8Qiqwb0etSWNla9YOpbzEp0MRKfAL0/W2sa1E6Zt6Xr4hIMagzgIiIBEqBRkREAqVAIyIigVKgERGRQMUs0JhZEzP70MyW+8+NI+Q51szmm9lCM/vWzMaHbTvVzBab2Qoze9JMNxKIiFREsTyiSQHSnHOdgDR/Pb8M4AznXA/gdCDFzFr72/4JjAM6+Y9hwVdZRERKKpaBZiTwvL/8PDAqfwbn3GHn3CF/tRZ+fc2sFdDAOfelc84BL0QqLyIisRfLQNPCOZcB4D83j5TJzI4xs0XAT8BfnHMbgDbAurBs6/y0SOXHmVm6maVv2bKlXBsgIiJFC/SGTTObCbSMsGlCcffhnPsJ6O6fMnvHzN4k8m25EceFcM5NBCYCJCcnBzV2hIiIRBFooHHODYm2zcw2mVkr51yGfypscxH72mBm3wL9gc+BtmGb2wIbyqPOIiJSvmJ56uxdYKy/PBaYkj+DmbU1s9r+cmOgH/C9f6ptj5n18XubXRGpvIiIxF4sA00qMNTMlgND/XXMLNnMJvl5TgTmmtk3wCfAI865xf62a4FJwApgJTD9aFZeRESKx1xgQ15XPMnJyS49PT3W1RARqVTMbL5zLrm05TUygIiIBEqBRkREAqVAIyIigVKgERGRQCnQiIhIoBRoREQkUAo0IiISKAUaEREJlAKNiIgESoFGREQCpUAjIiKBUqAREZFAKdCIiEigFGhERCRQCjQiIhIoBRoREQmUAo2IiARKgUZERAKlQCMiIoFSoBERkUAp0IiISKAUaEREJFAKNCIiEigFGhERCVRMAo2ZNTGzD81suf/cOEKeY81svpktNLNvzWx82LaPzex7f9tCM2t+dFsgIiLFFasjmhQgzTnXCUjz1/PLAM5wzvUATgdSzKx12PZLnXM9/Mfm4KssIiKlEatAMxJ43l9+HhiVP4Nz7rBz7pC/Wgud5hMRqZRi9eXdwjmXAeA/Rzz1ZWbHmNki4CfgL865DWGbJ/unze4yM4v2QmY2zszSzSx9y5Yt5dkGEREphsACjZnNNLMlER4ji7sP59xPzrnuwPHAWDNr4W+61Dl3MtDff1xeyD4mOueSnXPJzZo1K0uTRESkFBKC2rFzbki0bWa2ycxaOecyzKwVUOg1FufcBjP7Fi+ovOmcW++n7zGz/wC9gRfKsfoiIlJOYnXq7F1grL88FpiSP4OZtTWz2v5yY6Af8L2ZJZhZkp9eAzgfWHJUai0iIiUWq0CTCgw1s+XAUH8dM0s2s0l+nhOBuWb2DfAJ8IhzbjFex4AZ/rWbhcB64Jmj3QARESkec87Fug5HTXJysktPT491NUREKhUzm++cSy5teXUZFhGRQCnQiIhIoBRoREQkUAo0IiISKAUaEREJlAKNiIgESoFGREQCpUAjIiKBUqAREZFAKdCIiEigFGhERCRQCjQiIhIoBRoREQmUAo2IiARKgUZERAKlQCMiIoFSoBERkUAp0IiISKAUaEREJFAKNCIiEigFGhERCZQCjYiIBEqBRkREAqVAIyIigYpZoDGzJmb2oZkt958bF5K3gZmtN7OnwtJONbPFZrbCzJ40Mzs6NRcRkZKI5RFNCpDmnOsEpPnr0dwPfJIv7Z/AOKCT/xgWRCVFRKRsYhloRgLP+8vPA6MiZTKzU4EWwAdhaa2ABs65L51zDnghWnkREYmtWAaaFs65DAD/uXn+DGYWB/wN+GO+TW2AdWHr6/y0AsxsnJmlm1n6li1byqXiIiJSfAlB7tzMZgItI2yaUMxdXAdMc879lO8STKTrMS7SDpxzE4GJAMnJyRHziIhIcAINNM65IdG2mdkmM2vlnMvwT4VtjpCtL9DfzK4D6gE1zWwv8ATQNixfW2BDOVZdRETKSSxPnb0LjPWXxwJT8mdwzl3qnGvnnGsP3AK84JxL8U+17TGzPn5vsysilRcRkdiLZaBJBYaa2XJgqL+OmSWb2aRilL8WmASsAFYC04OqqIiIlJ55nbaqh+TkZJeenh7raoiIVCpmNt85l1za8hoZQEREAqVAIyIigVKgERGRQCnQiIhIoBRoREQkUAo0IiISKAUaEREJlAKNiIgESoFGREQCVaxAY2bHmVktf3mgmV1vZo2CrZqIiFQFxT2ieQvINrPjgX8DHYD/BFYrERGpMoobaHKcc1nAhcDjzrmbgFbBVUtERKqK4gaaTDMbgzec/3t+Wo1gqiQiIlVJcQPNVXiTkD3onFttZh2Al4KrloiIVBXFmmHTObcUuB7AzBoD9Z1zqUFWTEREqobi9jr72MwamFkT4Btgspk9GmzVRESkKijuqbOGzrndwM+Byc65U4EhwVVLRESqiuIGmgQzawVcxJHOACIiIkUqbqC5D5gBrHTOfWVmHYHlwVVLRESqiuJ2BngDeCNsfRXwi6AqJSIiVUdxOwO0NbO3zWyzmW0ys7fMrG3QlRMRkcqvuKfOJgPvAq2BNsD//DQREZFCFTfQNHPOTXbOZfmP54BmAdZLRESqiOIGmq1mdpmZxfuPy4BtQVZMRESqhuIGmqvxujZvBDKA0XjD0pSKmTUxsw/NbLn/3LiQvA3MbL2ZPRWW9rGZfW9mC/1H89LWRUREglWsQOOcW+ucu8A518w519w5Nwrv5s3SSgHSnHOdgDR/PZr7gU8ipF/qnOvhPzaXoS5FOpSVzcHM7CBfQkSkyirLDJs3l6HsSOB5f/l5YFSkTGZ2KtAC+KAMr1VmQx79hC53vR/LKoiIVFplCTRWhrItnHMZAP5zgVNfZhYH/A34Y5R9TPZPm91lZlHrYmbjzCzdzNK3bNlSqsr+tP1AqcqJiEgxb9iMwhW20cxmAi0jbJpQzP1fB0xzzv0UIY5c6pxbb2b18Wb/vBx4IWIlnZsITARITk4utM4iIlL+Cg00ZraHyAHFgNqFlXXORR1007/ps5VzLsMfQy3SNZa+QH8zuw6oB9Q0s73OuRTn3Hr/NfaY2X+A3kQJNOXp/SUZDOumiUVFREqi0FNnzrn6zrkGER71nXNlORp6F2+2TvznKRFe+1LnXDvnXHvgFuAF51yKmSWYWRKAmdUAzgeWlKEuxTb+pQUczso5Gi8lIlJllOUaTVmkAkPNbDkw1F/HzJLNbFIRZWsBM8xsEbAQWA88E2Rlw6VOX3a0XkpEpEow56rPZYvk5GSXnp5e4nLtU6bmWe/VrhH/va5feVVLRKRCM7P5zrnk0paP1RFNpbZg7c5YV0FEpNIoy3WWamPBXUOZviSDLi3r8+vn02nZIDHWVRIRqTQUaIqhSd2aXHr6sQAc16we83/cweY9B2leXwFHRKQoOnVWQsO7ebcGTV+8McY1ERGpHBRoSij3yOaVeWtjXBMRkcpBgaaEateMB2DZxj2s3bY/xrUREan4FGjKIOW/i2JdBRGRCk+BphTm3D4YgC9Wau43EZGiKNCUQsuGidT1T6GJiEjhFGhKqWOzegD8sGlPjGsiIlKxKdCU0q3DTgBg/Q7NVSMiUhgFmlLq2qoBAH98Ux0CREQKo0BTSk3q1gRg695DMa6JiEjFpkBTSuGzft495ahMhyMiUikp0JTBk2N6AvDClz/GuCYiIhWXAk0ZXHBK69Dy12t3xLAmIiIVlwJNGT131WkAXPh/X8S4JiIiFZMCTRmd1r5JaPn0h2bGsCYiIhWTAk0Z1a2VwFX92gOwabd6oImI5KdAUw7+9LOT+HnPNgAczMyOcW1ERCoWBZpyUqeWN/ZZl7veZ8/BzBjXRkSk4lCgKSe3DusSWj75ng/YdygrhrUREak4FGjKSYPEGnRMqhtaP+lPM2JYGxGRiiMmgcbMmpjZh2a23H9uHCVftpkt9B/vhqV3MLO5fvnXzKzm0at9dB/dMjB0rQbQDJwiIsTuiCYFSHPOdQLS/PVIDjjneviPC8LS/wI85pffAfw62OoW36MX9+DKM9oDcN9738a2MiIiFUCsAs1I4Hl/+XlgVHELmjfI2NnAm6UpfzTcc8FJAMz8bjMfLduEcy7GNRIRiZ1YBZoWzrkMAP+5eZR8iWaWbmZzzCw3mDQFdjrncq+2rwPaRC4eOx2beddrrn4unZPv+YB5q7czbXGGgo6IVDsJQe3YzGYCLSNsmlCC3bRzzm0ws47AR2a2GNgdIV/Ub28zGweMA2jXrl0JXrpsPvrDQNqnTAVg76EsLvrXlwDcNKQzNwzpBEBmdg4JcZZnJGgRkaomsCMa59wQ51y3CI8pwCYzawXgP2+Oso8N/vMq4GOgJ7AVaGRmuUGyLbChkHpMdM4lO+eSmzVrVm7tK441qefRplHtPGmPzfyBrOwcMrNz6DRhOh1un8aKzXuPar1ERI6mWJ06excY6y+PBabkz2Bmjc2slr+cBPQDljrv3NMsYHRh5SuK2bcOYlSP1vQOGxPt+AnT6TRhemh9yKOfsHnPwdD6/sNZDHx4FvN/3H5U6yoiEgSLxTUDM2sKvA60A9YCv3TObTezZGC8c+4aMzsD+BeQgxcQH3fO/dsv3xF4FWgCfA1c5pwrcqCx5ORkl56eHkibimPz7oP0figt4rZ/XtqL7zftYeZ3m1iy/sjZwTWp5x2t6omIRGRm851zyaUuX50uTsc60AC88OUa7p7idXu+52ddubBXW06594Oo+cf2PZZ7R3Y7SrUTESmorIEmsM4AEtkVfdtzRd/2xc7//Jc/cnnfYzm+eX2cc3S4fRqgIx0RqTw0BE0FsPKhEaT94SxWPjQilPb+jf1Dy0Me/ZS/py3ny5XbQmm513S+XruD9ilT2b7v8NGrsIhICejUWQXz0/b91EyIo0WDRIBQF+niePDCbpx3cisa1akQI/KISBVR1lNnOqKpYI5pUicUZIDQcDbFMeHtJfS470OGPzGb9ilTWbN1XwA1FBEpGQWaCu6eC05i6X3nMumKIz8mnr6sV54853dvlWf9uwyv19rARz7mnne/5c356ziUlc2OfYf51TNzuP6Vr4OvuIiIT6fOKpG07zZxyjGNSKpXC4B5q7dzUusG1K2VwIHD2Yx4cjarCzmK6dWuEQvW7gTgyTE9uf6Vr7nktGNI/UV3gNDwOGbGrv2Z1E9M4O8freCxmT+w+J5zqJ9YI+AWikhFpO7NJVDZA01xfLZ8K51a1OOpj1bw4pwfS1y+f6ckEuKMWd9voWZCHIezckLbfn1mB+46v2t5VldEKgEFmhKoDoEm3Dc/7WTDzgO0alSbUf/4vFz2+f6N/fl+4x7+9O63fH3X0BKP07Z2237ue28p1w8+nu5tG5VLnUQkWAo0JVDdAk2495ds5NnPV/PoRacwZ9V2bnnjGzo2q8uqLWXrMBDpfp7cnnKzbx1EnZrxNK1Xi4dnLGP6ko15Xu+pX/Xk7C7NqV0jXgOLilRgCjQlUJ0DTX4Lf9rJSa0bsOtAJskPzOTzlLMB6Jf6EV1bNeCd3/bjptcWMnVxBtNv6M+O/Yf51TNzC+zntmFduHbgcaH1VVv2cvbfPsmTZ/SpbXlz/rqodbl+cCduHtq50PoezMxmx/7DtGpYu8C2Xfsz2Z+Zxf7D2XRoWpe4OAUtkfKkQFMCCjRlE+2enim/7ccpxzRiyfpdnP/3z4rcz4u/7s3l/56XJ231n0dEPKoZ8cRslmbspmNSXVZt3ceKB4eTEJ+3s+Tof35B+o87QuvL7h9GYo344jRJRIpB99HIUfPWtX0BeO6q0/j6rqGh9NveWsTqrfsKBJlbzsl7lHLbsC7Mv3MI/Ts1Y/mDw/Ns+3ZDwWmGnHMs9btqr/J70/24fT8AG3cd5NJJczjhzul5ggxAl7veL03zRCQgOqKRUtt9MJPu9xQcEPSTPw7k2KbeDKNTFq7nbx/8wNTrz4zYPfqTH7Yw9lnv6Cb/9Z5vN+zivCcLHiEtu39YxGDSplFt1u88AHgznE67vn+5Htns2p9Jg9oJup4k1Y6OaCRmGiTW4PdnH58nrXf7JqEgAzCyRxs+vXVQ1HtwzjiuaWg590bTXAt/2hmxzGtf/VQgbcm95/J5ytm0aODdY7Rqyz5mfrepeA2JIjvH0T5lKqnTl/HVmu2cct8HDH9idpn2KVIdafRmKZM/nHMCb81fx4ZdB5l+Q39ObNWgROVrhF1vGf7EbM7u0pxlGbvZsOvIRHC511zufGcxL81Zy5/e9aZZeP03fUM3rObqmFSPTbu9qYlenfcT53dvnef11m7bz6qtexl4QvNC67Vo3U4ueMrrEv70Jyt5+pOVXl027ilR+0REgUbKwRe3Dy5T+Sm/7cdI/z6fj5YVnNU79/TXHSNO5KU5a0PpvTs0KZB34hWncu1LC/hsxVY+W7GVzXsO0ry+N3ZcTo5jwMOzAEi/cwhJ9Wqxbe8hTn1gJgCX9zm2WDe55uQ49WwTKQGdOpOYO7lNQ/4QpXvzdWFdp+vUTGD2rYMAePbKyKeL6yfW4KVrTufUYxsD0PvBNHJyvOuQuR0JAJIfmEmP+z4IBRmgyCAz5ETvKGjeGk2xLVISCjQSc3Fxxu8Hd+LVcX0AuGlIZ2bdMpDVfx7BrcO65Ml7TJM6rEk9j7O7tCh0n+GDkN7yxjcAvPP1+jx5du7PLHQfI3u0ZvWfR5BUrxZ/Hd2dm4eeAMC0xRkF8h7MzOZgZnah+4tk36Esetz3AbOXbylxWZHKQr3OpEL5aft+2jauXS49u9Zs3cfARz4uMt+y+4dxRupHbN93mKZ1a5L2h7NI+24zo3q2IT7sFFn4DKeh10g9j0NZ2Zxw55FecFN+24+WDRNp0SAxVKZT83p8ePNZoTwHDmezcsvePF3CNWuqVFSaylmqlGOa1Cm3fR3bNPK+cm8OTftuE9v2HSaxRjzz7xzC7oNZNKzt9Y77xaltC5SLFPz+M3ctd7y9OE9a7vWmlQ+NYOQ/vECyfPNeXpm3ljG920XtFi5SVenUmVRZZsYLV/fmlLYNQ2mPX9wjFDAGn9iCi5KPCeXNDTKFyb1Okyt/kAl33B3TWLL+SJft2/+7mBe/XEPP+z6MmH/ZxoI3rUZzOCuH/Yezip1fJJYUaKRKG9C5GS9eczqDuzRnzu2DGdWzTZn298wVyfz+7OOZceOAPOm3DevCmtTzmH3roAIBq1bCkX+zu6Z8S7bfOaF/pyRe/01fnhzTEyDqjKi5nRnC/ezvn9H17hmFBjqRikLXaERKKXzst/zXV6YuyuC3/1nAWZ2b8fzVvQucLjuhRX2m3dCf+Dhj/c4D9Ev9qMB+Vmzew8ffb+GBqd/RrU2D0NHRX0d359Y3F4XyrXxoBPFxxqTZq8hxjtM7NGXkPz7n67uG0rhuzUDaLtWLBtUsgUiBJjMzk3Xr1nHw4MEopaSkEhMTadu2LTVqVO0ZOQ9mZtPlrveZc/tgWjZMLDL/wIdnsWbbfm4ddgLXDTwyokJ4J4Pzu7fiqV/1In3NdkY//WWZ6xhtsFKRkqiUgcbMmgCvAe2BNcBFzrkdEfJlA7nnBtY65y7w058DzgJ2+duudM4tLOp1IwWa1atXU79+fZo2bap/yHLgnGPbtm3s2bOHDh06xLo6lcaLc37krneWAPDSr0/nsn8XnJIhv/O6t2LqooJdrcP95Rcnc/Fp7cqljlJ9VdaxzlKANOdcJyDNX4/kgHOuh/+4IN+2P4ZtKzLIRHPw4EEFmXJkZjRt2lRHiCV0eZ9jQ8vhQebck1rw3FWnkX7nEJbdP4xZtwzkmSuSOblNQx4e3Z3X/HuPcvVq581a2rKBd4R121uLOXD4yP09k2av4suV20pVxykL19M+ZWroGpNIccWqe/NIYKC//DzwMXBbjOqiIFPO9H6WzsvXnM6lk44EmV7tGvGvy/P+iOyQVJcOSXUZ2tW7YfX0jk157OJTMCxPR4fDWTl0vnM64PWMe+ziHuw+mMkDU78L5cm9tgPeacBpizO4sGebPJ+fc47HZi7n8j7HcsOr3u+537w4n0ljS/3jVqqhWB3RtHDOZQD4z9FGOEw0s3Qzm2Nmo/Jte9DMFpnZY2ZWK9DaihwF/Y5PokvL+oA3Odzrv+lbrHIX9mxboDddzYS4UPm3/RER/jTl2zx5nvpoBQDLN+2hy13vc/Pr39Dh9mmc9uBMnHPM/3E7HW6fxpNpyzntwSND9TRI1O13UjKB/cWY2UygZYRNE0qwm3bOuQ1m1hH4yMwWO+dWArcDG4GawES8o6H7otRjHDAOoF27ineuetu2bQwe7A1KuXHjRuLj42nWrBkA8+bNo2bNonsNXXXVVaSkpHDCCScEWlcJ3vv5uk2XRfigo+E95G4Y3Ikn0pazZe9Btu87zNDHPs1TbsueQwVGQAi3Zlvkbtj5Ld2wm5YNE2minm/VXmCBxjk3JNo2M9tkZq2ccxlm1gooOGSvt48N/vMqM/sY6AmszD0aAg6Z2WTglkLqMREvGJGcnFzhTi43bdqUhQu9UxL33HMP9erV45Zb8jbHOYdzjri4yAegkydPDryeUnXcNLQzT6Qt56U5a/OMhh3NLed05pEPfgBgcJfmpC3bzMHM7EInlcsNbI3r1ODru8/Jsy23l905XVsw8QqdgqsOYnUM/C4wFkj1n6fkz2BmjYH9zrlDZpYE9AP+6m/LDVIGjAKWlEel7v3ftyyNMKVwWXRt3YA//eykEpdbsWIFo0aN4swzz2Tu3Lm899573HvvvSxYsIADBw5w8cUXc/fddwNw5pln8tRTT9GtWzeSkpIYP34806dPp06dOkyZMoXmzQufe0WqrtuHd+HP05eF1n94YHjEfJOuSOakNg3o++eP8qR/dtsg2jauw6iebagRH8dF//K6XE94ewl/u+iUAvtxznEgbHDRHfszcc5hZixet4vvN+0JDXL6wdJNoYCkbthVW6wCTSrwupn9GlgL/BLAzJKB8c65a4ATgX+ZWQ7etaRU59xSv/zLZtYMMGAhMP5oN+BoWLp0KZMnT+bpp58GIDU1lSZNmpCVlcWgQYMYPXo0Xbt2zVNm165dnHXWWaSmpnLzzTfz7LPPkpISrVOfVHW/Oes4zjmpJQczs/NMSrcm9TxSpy/j+S/W8IdzOjPE71ywJvU8nHPM/G4zg7s0D82707axN27cm+PP4LQHZ/LWgnWs37mfOauKnjKhw+3T8kyzHS1PeOcEqVpiEmicc9uAArNlOefSgWv85S+Ak6OUPzuIepXmyCNIxx13HKeddlpo/ZVXXuHf//43WVlZbNiwgaVLlxYINLVr12b4cO9X66mnnsrs2Zp6uLrrkFQ3YnrK8C6kDO9SIN3MQr3a8mtWvxbxcUZ2jisyyDx4YTcmvO2dbMgfZEaf2pY356/Lk3bWw7P47LZA/rUlxtR9pAKrW/fIF8Ty5ct54oknmDdvHo0aNeKyyy6LeK9KeOeB+Ph4srI08KKUr+k39OecfB0IIrk4+Rg27DzAP2Z502Bf1a89V/frgJl3hHTT0M5Mmr2KhDjjmdmrWbfjQOg0W2HuefdburVpyOgII2xLxaRAU0ns3r2b+vXr06BBAzIyMpgxYwbDhg2LdbWkGurcoj5fpJzND5v28MXKbYw/6zjmrd7O+Jfmk/aHs5i+OIPubRuREB/HLeecQI9jGpOd4xjWLW8n1DaNaofOInyXsYfPVmylw+3TeOX/9aHvcU0jvvbkz1fz3BdrAPIEmv2Hs+h69wzuH9WNy/scy64DmdSrlaBTcRWEAk0l0atXL7p27Uq3bt3o2LEj/fr1i3WVpBpr3ag2rRvVZuAJXkeTYd1ahgYE/d3ZnUL5CjsNF+6JS3qEptUe88ycYtUhKzuHhPg49h7KotufZgBw1ztLeHnOjyzbuAfwBiA9t2tLGtap2uPuVXTVflDN7777jhNPPDFGNaq69L5KST364Q88mba82PmT6tVi695Dxcqr2UvLprKOdSYiksfNQzsz944CfYTySIgz/ve7MwEKBJnfDTo+UpFi27r3EHsPZbF176Fyv82hutOpMxGpMFo0SCxw9LFz/2Hq1Eygpj+B3MGw+3TAu4m0VaNEbh7amcv6HEufP6dxzZkdyMpxoes5Z/7lozw92h6a9h0TP13F0vvOpU7NBDKzc0h+YGae/Y4/6zh+M6AjAKu37ePn//dFaNsb4/tyWvsmSPHo1JlO8QRC71whZyoAABINSURBVKsE6c3564iP88Z5yy8nx4Xu//nvgnXc/Lp3g+iqh0YQF2fs3H+YHlGm0y6J3P1VBzp1JiLVzuhT20YMMkCeL/8LTmkdWl6asZsNOw+US5AB6HLX++WyH4APvt1I+5SpLN+0p9z2WZEo0IhIlZUQH8fU671rOjO+3cgZqUeG2Ml/I2urhoncP6obAKN6eAGqZnxc6NrPw6O7syb1PGbfOgiAw9k5ZGXnlKg+t7zxDe1TpnLgcDb7D2fx6ry1tE+ZyrgX5wPwfx+vLEUrKz5doxGRKq1jUj0A/u5PiwDeyNa50yjk5DiWbdxD19beED25k9A99POTqVPT+4q85dwjI6Mf06ROaPn4CdOZceMA6tSMZ/XWfQzo3Iz3l2xk76GsPPf5dJ4wncNhQWn+jzsizqLaplHtMre3ItIRTYwNHDiQGTNm5El7/PHHue6666KWqVfP+8fZsGEDo0ePjrrf/Nej8nv88cfZv39/aH3EiBHs3LmzuFUXqRRq18w7yvT53VvlmesnLs5CQSZcbpApyrmPf0r/v87iimfn0T5lKuNfmh8aODQnx/G7/yzIE2SAqFN1b9pdNWemVaCJsTFjxvDqq6/mSXv11VcZM2ZMkWVbt27Nm2++WerXzh9opk2bRqNGjUq9P5HK4O9jepZ5H8vuH8a5JxV+I+pNry2k4x3TeG9RRqH5wuvzxvx1bN17iH6pH/HFyq0R82/be4gT7pzOJRO/JKeSTKutU2fhpqfAxsXlu8+WJ8Pw1KibR48ezZ133smhQ4eoVasWa9asYcOGDfTo0YPBgwezY8cOMjMzeeCBBxg5cmSesmvWrOH8889nyZIlHDhwgKuuuoqlS5dy4okncuDAkUEMr732Wr766isOHDjA6NGjuffee3nyySfZsGEDgwYNIikpiVmzZtG+fXvS09NJSkri0Ucf5dlnnwXgmmuu4cYbb2TNmjUMHz6cM888ky+++II2bdowZcoUateumof7UnV8NWEIpz04kzG925XLdASJNeL51+XJdPvTDPYeijyeYO7MprneGN+XnBxH/cQajHjSG+w2tyt3+6Z1+dlTnwGEuln/6pm5ebp6f712B3+evoxaCXEcysphzqrtzF29vcBwPQczs+ly1/uMG9CRGwZ34vX0n7iib/uYDsejQBNjTZs2pXfv3rz//vuMHDmSV199lYsvvpjatWvz9ttv06BBA7Zu3UqfPn244IILov6T/POf/6ROnTosWrSIRYsW0atXr9C2Bx98kCZNmpCdnc3gwYNZtGgR119/PY8++iizZs0iKSkpz77mz5/P5MmTmTt3Ls45Tj/9dM466ywaN27M8uXLeeWVV3jmmWe46KKLeOutt7jssssCfY9EyqpZ/VqBjA6QfucQnvl0Ff9vQEcW/LiDjs3qUadWPN3v+SCU59VxfejTMW8weP/G/rRskBhaP7ltwzyjXYfyLclgWLdWAFwYdh9Prpfm/EjnFvXYvu8wHZLqkhAfxx1vez+WJ366iomfrgKgSd2ajOzRpkD5o0WBJlwhRx5Byj19lhtonn32WZxz3HHHHXz66afExcWxfv16Nm3aRMuWkWbHhk8//ZTrr78egO7du9O9e/fQttdff52JEyeSlZVFRkYGS5cuzbM9v88++4wLL7wwNHr0z3/+c2bPns0FF1xAhw4d6NGjB+BNQ7BmzZpyehdEKp/EGvH8frA3ttsZxx/5wTZvwmCmLcogqX6tAkEGoEvLgteELk4+pkCgGf/SAp6+rBdrtu0vkB9g6uIMpi72Ts2d0rYhN59zAv9dsL5Avnv/tzSmgUbXaCqAUaNGkZaWFpo9s1evXrz88sts2bKF+fPns3DhQlq0aBFxWoBwkY52Vq9ezSOPPEJaWhqLFi3ivPPOK3I/hd3EW6tWrdCypiEQiax5/USu7NeB87u3LjqzLyH+yNfx8c3rhZbHv7SAVH+W1L+P6cmAzs34asKQAuW/WbeLsc/OA7xu2eH72L7vcKH/10FToKkA6tWrx8CBA7n66qtDnQB27dpF8+bNqVGjBrNmzeLHH38sdB8DBgzg5ZdfBmDJkiUsWrQI8KYXqFu3Lg0bNmTTpk1Mnz49VKZ+/frs2VPwBrEBAwbwzjvvsH//fvbt28fbb79N//79y6u5IhLFp38cRKuGiUy/oT/f3H1Oge2DT2zOC1f3pln9WpxyTPSOO8vuH8bMm89ixYPD6eQHnAVrY9ejVIGmghgzZgzffPMNl1xyCQCXXnop6enpJCcn8/LLL9OlS8GZEMNde+217N27l+7du/PXv/6V3r17A3DKKafQs2dPTjrpJK6++uo80wuMGzeO4cOHM2jQoDz76tWrF1deeSW9e/fm9NNP55prrqFnz7L31BGRwrVrWocvbx9Mjfg4GtapkSeYLL7nnDxdridfeWT23fDBSFOGdwmNjpAQH8c/L+vFH4Z2ple72PUo1VhnGpMrEHpfRcoud1y2E1rUZ8ZNAwrN++b8ddSrFR/qPFCeyjrWmToDiIhUUI3q1Cx2b7mKPLW1Tp2JiEigFGgovJeVlJzeTxEJV+0DTWJiItu2bdOXYzlxzrFt2zYSExOLziwi1UK1v0bTtm1b1q1bx5YtW2JdlSojMTGRtm0r7vliETm6YhJozKwJ8BrQHlgDXOSc2xEhXztgEnAM4IARzrk1ZtYBeBVoAiwALnfOHS5NXWrUqEGHDh1KU1RERIohVqfOUoA051wnIM1fj+QF4GHn3IlAb2Czn/4X4DG//A7g1wHXV0RESilWgWYk8Ly//DwwKn8GM+sKJDjnPgRwzu11zu03b5yVs4E3CysvIiIVQ6wCTQvnXAaA/9w8Qp7OwE4z+6+ZfW1mD5tZPNAU2Omcyx1kax0QdbQ4MxtnZulmlq7rMCIiR19g12jMbCYQaajhCcXcRQLQH+gJrMW7pnMl8G6EvFG7jDnnJgIT/TptMbPCBw2LLgmIPBNR1Vdd215d2w1qu9qe17Fl2WlggcY5V3B4UZ+ZbTKzVs65DDNrxZFrL+HWAV8751b5Zd4B+gDPAo3MLME/qmkLbChmnZqVtB1hdU4vyxAMlVl1bXt1bTeo7Wp7+YrVqbN3gbH+8lhgSoQ8XwGNzSw3OJwNLHXeDS+zgNFFlBcRkQogVoEmFRhqZsuBof46ZpZsZpMAnHPZwC1AmpktBgx4xi9/G3Czma3Au2bz76NcfxERKaaY3EfjnNsGDI6Qng5cE7b+IVBgKkj/dFrvIOsYwcSj/HoVSXVte3VtN6jt1VUgba9W0wSIiMjRV+3HOhMRkWAp0IiISKAUaIrBzIaZ2fdmtsLMog2XU2mY2TFmNsvMvjOzb83sBj+9iZl9aGbL/efGfrqZ2ZN++xeZWa+wfY318y83s7HRXrMiMbN4/ybg9/z1DmY212/Da2ZW00+v5a+v8Le3D9vH7X7692Z2bmxaUjJm1sjM3jSzZf5n37cafeY3+X/rS8zsFTNLrKqfu5k9a2abzWxJWFq5fc5mdqqZLfbLPGlmVmSlnHN6FPIA4oGVQEegJvAN0DXW9Spjm1oBvfzl+sAPQFfgr0CKn54C/MVfHgFMx+v51weY66c3AVb5z4395caxbl8x2n8z8B/gPX/9deASf/lp4Fp/+TrgaX/5EuA1f7mr/3dQC+jg/33Ex7pdxWj388A1/nJNoFF1+MzxRg5ZDdQO+7yvrKqfOzAA6AUsCUsrt88ZmAf09ctMB4YXWadYvykV/eG/oTPC1m8Hbo91vcq5jVPwupl/D7Ty01oB3/vL/wLGhOX/3t8+BvhXWHqefBXxgXeDbxrefVnv+f8sW/HG1cvzeQMzgL7+coKfz/L/DYTnq6gPoIH/ZWv50qvDZ94G+Mn/0kzwP/dzq/LnjjcyfnigKZfP2d+2LCw9T75oD506K1ruH2muQsdWq2z80wI9gblEH4Mu2ntQGd+bx4FbgRx/vbCx80Lt87fv8vNXxnZ3BLYAk/3ThpPMrC7V4DN3zq0HHsEbyioD73OcT/X43HOV1+fcxl/On14oBZqiRTr/WCX6hJtZPeAt4Ebn3O7CskZIc4WkV0hmdj6w2Tk3Pzw5QlZXxLZK1W5fAt7plH8653oC+4g+PQdUobb71yNG4p3uag3UBYZHyFoVP/eilLStpXoPFGiKtg5v4rVcxR5brSIzsxp4QeZl59x//eRN5o09h+Udgy7ae1DZ3pt+wAVmtgZv4ryz8Y5wGplZ7s3L4W0Itc/f3hDYTuVrN3h1Xuecm+uvv4kXeKr6Zw4wBFjtnNvinMsE/gucQfX43HOV1+e8zl/On14oBZqifQV08nuo1MS7OBhpBOlKw+8l8m/gO+fco2Gboo1B9y5whd9DpQ+wyz/8ngGcY2aN/V+N5/hpFZJz7nbnXFvnXHu8z/Ej59ylRB87L/z9GO3nd376JX7vpA5AJ7wLpBWWc24j8JOZneAnDQaWUsU/c99aoI+Z1fH/9nPbXuU/9zDl8jn72/aYWR//vbyC4ow1GeuLVpXhgdcz4we8XiYTYl2fcmjPmXiHu4uAhf5jBN556DRguf/cxM9vwD/89i8GksP2dTWwwn9cFeu2leA9GMiRXmcd8b4wVgBvALX89ER/fYW/vWNY+Qn++/E9xeh1UxEeQA8g3f/c38HrTVQtPnPgXmAZsAR4Ea/nWJX83IFX8K5FZeIdgfy6PD9nINl/H1cCT5Gvg0mkh4agERGRQOnUmYiIBEqBRkREAqVAIyIigVKgERGRQCnQiIhIoBRopEoxs2wzW2hm35jZAjM7o4j8jczsumLs92MzSy6/mlZ+ZvacmY0uOqdUdwo0UtUccM71cM6dgjcI4p+LyN8Ib7TeCinsznWRSkuBRqqyBsAO8MZ1M7M0/yhnsZmN9POkAsf5R0EP+3lv9fN8Y2apYfv7pZnNM7MfzKy/nzfezB42s6/8+Tx+46e3MrNP/f0uyc0fzszWmNlf/H3OM7Pj/fTnzOxRM5sF/MWfS+Qdf/9zzKx7WJsm+3VdZGa/8NPPMbMv/ba+4Y9ph5mlmtlSP+8jftov/fp9Y2afFtEmM7On/H1M5cjAjCKF0q8lqWpqm9lCvLu7W+GNZwZwELjQObfbzJKAOWb2Lt7Akt2ccz0AzGw4MAo43Tm338yahO07wTnX28xGAH/CG0Pr13jDdpxmZrWAz83sA+DneEN2PGhm8UCdKPXd7e/zCrxx18730zsDQ5xz2Wb2d+Br59woMzsbeAHvLv+7/Nc+2a97Y79td/pl95nZbcDNZvYUcCHQxTnnzKyR/zp3A+c659aHpUVrU0/gBOBkoAXeMC7PFutTkWpNgUaqmgNhQaMv8IKZdcMbauMhMxuAN0VAG7wvy/yGAJOdc/sBnHPbw7blDj46H2++D/DGgOoedq2iId4YWF8Bz5o3eOk7zrmFUer7StjzY2Hpbzjnsv3lM4Ff+PX5yMyamllDv66X5BZwzu0wb4TqrnjBAbwJzr4EduMF20n+0ch7frHPgefM7PWw9kVr0wDgFb9eG8zsoyhtEslDgUaqLOfcl/4v/GZ4Y7k1A051zmWaN4JzYoRiRvRhzw/5z9kc+d8x4PfOuQIDS/pB7TzgRTN72Dn3QqRqRlnel69OkcpFqqsBHzrnxkSoT2+8ASUvAX4HnO2cG29mp/v1XGhmPaK1yT+S05hVUmK6RiNVlpl1wZuKexver/LNfpAZBBzrZ9uDN511rg+Aq82sjr+P8FNnkcwArvWPXDCzzmZW18yO9V/vGbyRsntFKX9x2POXUfJ8Clzq738gsNV58wd9gBcwctvbGJgD9Au73lPHr1M9oKFzbhpwI96pN8zsOOfcXOfc3XgzSR4TrU1+PS7xr+G0AgYV8d6IADqikaon9xoNeL/Mx/rXOV4G/mdm6XijVS8DcM5tM7PPzWwJMN0590f/V326mR0GpgF3FPJ6k/BOoy0w71zVFrxrPAOBP5pZJrAXbzj1SGqZ2Vy8H30FjkJ89+DNjLkI2M+R4d4fAP7h1z0buNc5918zuxJ4xb++At41mz3AFDNL9N+Xm/xtD5tZJz8tDfgGb3TnSG16G++a12K80cw/KeR9EQnR6M0iMeKfvkt2zm2NdV1EgqRTZyIiEigd0YiISKB0RCMiIoFSoBERkUAp0IiISKAUaEREJFAKNCIiEqj/D0crRZFhPWuVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:21:08.400771Z",
     "start_time": "2019-08-21T02:21:06.713776Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.fit_one_cycle(500, 2e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T11:15:02.866857Z",
     "start_time": "2019-08-15T11:15:02.810887Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.recorder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optional Fine tune regular fit "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:26:43.041291Z",
     "start_time": "2019-08-21T03:26:43.034173Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.callbacks.append(\n",
    "    ReduceLROnPlateauCallback(learner, monitor='train_loss', mode='min', factor=0.2, patience=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T06:19:15.311297Z",
     "start_time": "2019-08-21T03:27:35.491980Z"
    },
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#learner.opt_func = RAdam\n",
    "learner.fit(50, 1e-3)# , moms=(0.75,0.70))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.fit_one_cycle(300, 5e-4)#, moms=(0.75,0.70))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-10T11:24:45.530739Z",
     "start_time": "2019-08-10T11:24:44.737186Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_lr(show_moms=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T16:57:30.084191Z",
     "start_time": "2019-08-16T16:57:29.686756Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-16T16:57:43.396559Z",
     "start_time": "2019-08-16T16:57:41.386234Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.fit(10,1e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.model = learner.model.module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AtomTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (drop_emb): Dropout(p=0.0, inplace=False)\n",
       "    (layers): ModuleList(\n",
       "      (0): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (4): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (5): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (scalar): Conv1d(1033, 1, kernel_size=(1,), stride=(1,))\n",
       "  (magnetic): Conv1d(1024, 9, kernel_size=(1,), stride=(1,))\n",
       "  (dipole): Linear(in_features=29696, out_features=3, bias=True)\n",
       "  (potential): Linear(in_features=29696, out_features=1, bias=True)\n",
       "  (type_embedding): Embedding(9, 509)\n",
       "  (atom_embedding): Embedding(6, 512)\n",
       "  (dropout): Dropout(p=0, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ParallelTrainer\n",
       "learn: Learner(data=DataBunch;\n",
       "\n",
       "Train: LabelList (707619 items)\n",
       "x: ItemList\n",
       "1 5 atoms 4 couplings,2 5 atoms 4 couplings,3 5 atoms 4 couplings,4 5 atoms 4 couplings,6 4 atoms 3 couplings\n",
       "y: ScalarCouplingList\n",
       "4: 84.8076 * -11.2569 -11.2549 -11.2543,4: 84.8074 -11.2569 * -11.2542 -11.2548,4: 84.8093 -11.2549 -11.2542 * -11.2543,4: 84.8095 -11.2543 -11.2548 -11.2543,3: 32.6888 * -11.1867 -11.1757\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (78217 items)\n",
       "x: ItemList\n",
       "16 8 atoms 7 couplings,17 8 atoms 7 couplings,18 8 atoms 7 couplings,19 8 atoms 7 couplings,20 8 atoms 7 couplings\n",
       "y: ScalarCouplingList\n",
       "7: 83.5430 -2.3783 * -11.7004 -11.6979 3.2528 13.6913 3.2521,7: 83.5417 -2.3786 -11.7004 * -11.6996 13.6924 3.2525 3.2527,7: 83.5484 -2.3772 -11.6979 -11.6996 * 3.2524 3.2524 13.6921,7: -2.3788 83.5418 3.2528 13.6924 3.2524 * -11.7004 -11.6993,7: -2.3785 83.5430 13.6913 3.2525 3.2524 -11.7004 * -11.6976\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=DataParallel(\n",
       "  (module): DataParallel(\n",
       "    (module): AtomTransformer(\n",
       "      (transformer): Transformer(\n",
       "        (drop_emb): Dropout(p=0.0, inplace=False)\n",
       "        (layers): ModuleList(\n",
       "          (0): DecoderLayer(\n",
       "            (mhra): MultiHeadAttention(\n",
       "              (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (drop_att): Dropout(p=0.0, inplace=False)\n",
       "              (drop_res): Dropout(p=0.0, inplace=False)\n",
       "              (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            )\n",
       "            (ff): SequentialEx(\n",
       "              (layers): ModuleList(\n",
       "                (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                (1): Swish()\n",
       "                (2): Dropout(p=0.0, inplace=False)\n",
       "                (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                (4): Dropout(p=0.0, inplace=False)\n",
       "                (5): MergeLayer()\n",
       "                (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (1): DecoderLayer(\n",
       "            (mhra): MultiHeadAttention(\n",
       "              (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (drop_att): Dropout(p=0.0, inplace=False)\n",
       "              (drop_res): Dropout(p=0.0, inplace=False)\n",
       "              (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            )\n",
       "            (ff): SequentialEx(\n",
       "              (layers): ModuleList(\n",
       "                (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                (1): Swish()\n",
       "                (2): Dropout(p=0.0, inplace=False)\n",
       "                (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                (4): Dropout(p=0.0, inplace=False)\n",
       "                (5): MergeLayer()\n",
       "                (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (2): DecoderLayer(\n",
       "            (mhra): MultiHeadAttention(\n",
       "              (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (drop_att): Dropout(p=0.0, inplace=False)\n",
       "              (drop_res): Dropout(p=0.0, inplace=False)\n",
       "              (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            )\n",
       "            (ff): SequentialEx(\n",
       "              (layers): ModuleList(\n",
       "                (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                (1): Swish()\n",
       "                (2): Dropout(p=0.0, inplace=False)\n",
       "                (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                (4): Dropout(p=0.0, inplace=False)\n",
       "                (5): MergeLayer()\n",
       "                (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (3): DecoderLayer(\n",
       "            (mhra): MultiHeadAttention(\n",
       "              (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (drop_att): Dropout(p=0.0, inplace=False)\n",
       "              (drop_res): Dropout(p=0.0, inplace=False)\n",
       "              (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            )\n",
       "            (ff): SequentialEx(\n",
       "              (layers): ModuleList(\n",
       "                (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                (1): Swish()\n",
       "                (2): Dropout(p=0.0, inplace=False)\n",
       "                (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                (4): Dropout(p=0.0, inplace=False)\n",
       "                (5): MergeLayer()\n",
       "                (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (4): DecoderLayer(\n",
       "            (mhra): MultiHeadAttention(\n",
       "              (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (drop_att): Dropout(p=0.0, inplace=False)\n",
       "              (drop_res): Dropout(p=0.0, inplace=False)\n",
       "              (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            )\n",
       "            (ff): SequentialEx(\n",
       "              (layers): ModuleList(\n",
       "                (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                (1): Swish()\n",
       "                (2): Dropout(p=0.0, inplace=False)\n",
       "                (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                (4): Dropout(p=0.0, inplace=False)\n",
       "                (5): MergeLayer()\n",
       "                (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (5): DecoderLayer(\n",
       "            (mhra): MultiHeadAttention(\n",
       "              (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "              (drop_att): Dropout(p=0.0, inplace=False)\n",
       "              (drop_res): Dropout(p=0.0, inplace=False)\n",
       "              (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            )\n",
       "            (ff): SequentialEx(\n",
       "              (layers): ModuleList(\n",
       "                (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "                (1): Swish()\n",
       "                (2): Dropout(p=0.0, inplace=False)\n",
       "                (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "                (4): Dropout(p=0.0, inplace=False)\n",
       "                (5): MergeLayer()\n",
       "                (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (scalar): Conv1d(1033, 1, kernel_size=(1,), stride=(1,))\n",
       "      (magnetic): Conv1d(1024, 9, kernel_size=(1,), stride=(1,))\n",
       "      (dipole): Linear(in_features=29696, out_features=3, bias=True)\n",
       "      (potential): Linear(in_features=29696, out_features=1, bias=True)\n",
       "      (type_embedding): Embedding(9, 509)\n",
       "      (atom_embedding): Embedding(6, 512)\n",
       "      (dropout): Dropout(p=0, inplace=False)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=LMAEMaskedLoss(), metrics=[], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False), <class '__main__.ShowGraph'>], callbacks=[LMAEMetric\n",
       "learn: ...\n",
       "val_only: True, SaveModelCustomCallback\n",
       "learn: ...\n",
       "monitor: 👉🏻LMAE👈🏻\n",
       "mode: min\n",
       "every: improvement\n",
       "name: bestmodel], layer_groups=[Sequential(\n",
       "  (0): Dropout(p=0.0, inplace=False)\n",
       "  (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (5): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (6): Dropout(p=0.0, inplace=False)\n",
       "  (7): Dropout(p=0.0, inplace=False)\n",
       "  (8): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (9): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (10): Swish()\n",
       "  (11): Dropout(p=0.0, inplace=False)\n",
       "  (12): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (13): Dropout(p=0.0, inplace=False)\n",
       "  (14): MergeLayer()\n",
       "  (15): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (16): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (17): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (18): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (19): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (20): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (21): Dropout(p=0.0, inplace=False)\n",
       "  (22): Dropout(p=0.0, inplace=False)\n",
       "  (23): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (24): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (25): Swish()\n",
       "  (26): Dropout(p=0.0, inplace=False)\n",
       "  (27): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (28): Dropout(p=0.0, inplace=False)\n",
       "  (29): MergeLayer()\n",
       "  (30): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (31): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (32): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (33): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (34): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (35): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (36): Dropout(p=0.0, inplace=False)\n",
       "  (37): Dropout(p=0.0, inplace=False)\n",
       "  (38): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (39): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (40): Swish()\n",
       "  (41): Dropout(p=0.0, inplace=False)\n",
       "  (42): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (43): Dropout(p=0.0, inplace=False)\n",
       "  (44): MergeLayer()\n",
       "  (45): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (46): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (47): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (48): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (49): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (50): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (51): Dropout(p=0.0, inplace=False)\n",
       "  (52): Dropout(p=0.0, inplace=False)\n",
       "  (53): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (54): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (55): Swish()\n",
       "  (56): Dropout(p=0.0, inplace=False)\n",
       "  (57): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (58): Dropout(p=0.0, inplace=False)\n",
       "  (59): MergeLayer()\n",
       "  (60): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (61): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (62): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (63): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (64): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (65): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (66): Dropout(p=0.0, inplace=False)\n",
       "  (67): Dropout(p=0.0, inplace=False)\n",
       "  (68): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (69): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (70): Swish()\n",
       "  (71): Dropout(p=0.0, inplace=False)\n",
       "  (72): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (73): Dropout(p=0.0, inplace=False)\n",
       "  (74): MergeLayer()\n",
       "  (75): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (76): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (77): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (78): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (79): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (80): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (81): Dropout(p=0.0, inplace=False)\n",
       "  (82): Dropout(p=0.0, inplace=False)\n",
       "  (83): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (84): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (85): Swish()\n",
       "  (86): Dropout(p=0.0, inplace=False)\n",
       "  (87): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (88): Dropout(p=0.0, inplace=False)\n",
       "  (89): MergeLayer()\n",
       "  (90): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (91): Conv1d(1033, 1, kernel_size=(1,), stride=(1,))\n",
       "  (92): Conv1d(1024, 9, kernel_size=(1,), stride=(1,))\n",
       "  (93): Linear(in_features=29696, out_features=3, bias=True)\n",
       "  (94): Linear(in_features=29696, out_features=1, bias=True)\n",
       "  (95): Embedding(9, 509)\n",
       "  (96): Embedding(6, 512)\n",
       "  (97): Dropout(p=0, inplace=False)\n",
       ")], add_time=True, silent=False)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.callbacks.pop(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SaveModelCustomCallback\n",
       "learn: Learner(data=DataBunch;\n",
       "\n",
       "Train: LabelList (707619 items)\n",
       "x: ItemList\n",
       "1 5 atoms 4 couplings,2 5 atoms 4 couplings,3 5 atoms 4 couplings,4 5 atoms 4 couplings,6 4 atoms 3 couplings\n",
       "y: ScalarCouplingList\n",
       "4: 84.8076 * -11.2569 -11.2549 -11.2543,4: 84.8074 -11.2569 * -11.2542 -11.2548,4: 84.8093 -11.2549 -11.2542 * -11.2543,4: 84.8095 -11.2543 -11.2548 -11.2543,3: 32.6888 * -11.1867 -11.1757\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (78217 items)\n",
       "x: ItemList\n",
       "16 8 atoms 7 couplings,17 8 atoms 7 couplings,18 8 atoms 7 couplings,19 8 atoms 7 couplings,20 8 atoms 7 couplings\n",
       "y: ScalarCouplingList\n",
       "7: 83.5430 -2.3783 * -11.7004 -11.6979 3.2528 13.6913 3.2521,7: 83.5417 -2.3786 -11.7004 * -11.6996 13.6924 3.2525 3.2527,7: 83.5484 -2.3772 -11.6979 -11.6996 * 3.2524 3.2524 13.6921,7: -2.3788 83.5418 3.2528 13.6924 3.2524 * -11.7004 -11.6993,7: -2.3785 83.5430 13.6913 3.2525 3.2524 -11.7004 * -11.6976\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=AtomTransformer(\n",
       "  (transformer): Transformer(\n",
       "    (drop_emb): Dropout(p=0.0, inplace=False)\n",
       "    (layers): ModuleList(\n",
       "      (0): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (1): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (2): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (3): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (4): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (5): DecoderLayer(\n",
       "        (mhra): MultiHeadAttention(\n",
       "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (drop_att): Dropout(p=0.0, inplace=False)\n",
       "          (drop_res): Dropout(p=0.0, inplace=False)\n",
       "          (ln): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (ff): SequentialEx(\n",
       "          (layers): ModuleList(\n",
       "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (1): Swish()\n",
       "            (2): Dropout(p=0.0, inplace=False)\n",
       "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (4): Dropout(p=0.0, inplace=False)\n",
       "            (5): MergeLayer()\n",
       "            (6): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (scalar): Conv1d(1033, 1, kernel_size=(1,), stride=(1,))\n",
       "  (magnetic): Conv1d(1024, 9, kernel_size=(1,), stride=(1,))\n",
       "  (dipole): Linear(in_features=29696, out_features=3, bias=True)\n",
       "  (potential): Linear(in_features=29696, out_features=1, bias=True)\n",
       "  (type_embedding): Embedding(9, 509)\n",
       "  (atom_embedding): Embedding(6, 512)\n",
       "  (dropout): Dropout(p=0, inplace=False)\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=LMAEMaskedLoss(), metrics=[], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False), <class '__main__.ShowGraph'>], callbacks=[LMAEMetric\n",
       "learn: ...\n",
       "val_only: True, SaveModelCustomCallback\n",
       "learn: ...\n",
       "monitor: 👉🏻LMAE👈🏻\n",
       "mode: min\n",
       "every: improvement\n",
       "name: bestmodel], layer_groups=[Sequential(\n",
       "  (0): Dropout(p=0.0, inplace=False)\n",
       "  (1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (3): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (4): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (5): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (6): Dropout(p=0.0, inplace=False)\n",
       "  (7): Dropout(p=0.0, inplace=False)\n",
       "  (8): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (9): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (10): Swish()\n",
       "  (11): Dropout(p=0.0, inplace=False)\n",
       "  (12): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (13): Dropout(p=0.0, inplace=False)\n",
       "  (14): MergeLayer()\n",
       "  (15): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (16): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (17): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (18): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (19): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (20): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (21): Dropout(p=0.0, inplace=False)\n",
       "  (22): Dropout(p=0.0, inplace=False)\n",
       "  (23): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (24): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (25): Swish()\n",
       "  (26): Dropout(p=0.0, inplace=False)\n",
       "  (27): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (28): Dropout(p=0.0, inplace=False)\n",
       "  (29): MergeLayer()\n",
       "  (30): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (31): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (32): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (33): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (34): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (35): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (36): Dropout(p=0.0, inplace=False)\n",
       "  (37): Dropout(p=0.0, inplace=False)\n",
       "  (38): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (39): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (40): Swish()\n",
       "  (41): Dropout(p=0.0, inplace=False)\n",
       "  (42): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (43): Dropout(p=0.0, inplace=False)\n",
       "  (44): MergeLayer()\n",
       "  (45): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (46): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (47): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (48): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (49): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (50): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (51): Dropout(p=0.0, inplace=False)\n",
       "  (52): Dropout(p=0.0, inplace=False)\n",
       "  (53): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (54): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (55): Swish()\n",
       "  (56): Dropout(p=0.0, inplace=False)\n",
       "  (57): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (58): Dropout(p=0.0, inplace=False)\n",
       "  (59): MergeLayer()\n",
       "  (60): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (61): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (62): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (63): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (64): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (65): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (66): Dropout(p=0.0, inplace=False)\n",
       "  (67): Dropout(p=0.0, inplace=False)\n",
       "  (68): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (69): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (70): Swish()\n",
       "  (71): Dropout(p=0.0, inplace=False)\n",
       "  (72): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (73): Dropout(p=0.0, inplace=False)\n",
       "  (74): MergeLayer()\n",
       "  (75): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (76): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (77): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (78): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (79): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (80): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (81): Dropout(p=0.0, inplace=False)\n",
       "  (82): Dropout(p=0.0, inplace=False)\n",
       "  (83): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (84): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "  (85): Swish()\n",
       "  (86): Dropout(p=0.0, inplace=False)\n",
       "  (87): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "  (88): Dropout(p=0.0, inplace=False)\n",
       "  (89): MergeLayer()\n",
       "  (90): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (91): Conv1d(1033, 1, kernel_size=(1,), stride=(1,))\n",
       "  (92): Conv1d(1024, 9, kernel_size=(1,), stride=(1,))\n",
       "  (93): Linear(in_features=29696, out_features=3, bias=True)\n",
       "  (94): Linear(in_features=29696, out_features=1, bias=True)\n",
       "  (95): Embedding(9, 509)\n",
       "  (96): Embedding(6, 512)\n",
       "  (97): Dropout(p=0, inplace=False)\n",
       ")], add_time=True, silent=False)\n",
       "monitor: 👉🏻LMAE👈🏻\n",
       "mode: min\n",
       "every: improvement\n",
       "name: bestmodel"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.callbacks[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:22:55.624674Z",
     "start_time": "2019-08-21T02:21:08.404032Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with 👉🏻LMAE👈🏻 value: -3.0483055114746094.\n",
      "[-0.28589037, tensor(-3.0490, device='cuda:0'), tensor(-2.0154, device='cuda:0'), tensor(-3.5112, device='cuda:0'), tensor(-2.1933, device='cuda:0'), tensor(-3.4626, device='cuda:0'), tensor(-3.0657, device='cuda:0'), tensor(-3.5653, device='cuda:0'), tensor(-2.9282, device='cuda:0'), tensor(-3.6498, device='cuda:0')] tensor(-2.1933, device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "learner.to_fp32()\n",
    "\n",
    "val = learner.validate()\n",
    "val_lmae = val[2+type_to_tune]\n",
    "print(val, val_lmae)\n",
    "val = val[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:22:56.164688Z",
     "start_time": "2019-08-21T02:22:55.627089Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss-0.5834val-3.0490_lmae2-2.1933\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sub_fname = f'loss{learner.recorder.losses[-1]:.04f}val{val:.04f}_lmae{type_to_tune}{val_lmae:.04f}'\n",
    "except Exception as e:\n",
    "    sub_fname = f'val{val:.04f}_lmae{type_to_tune}{val_lmae:.04f}'\n",
    "learner.save(sub_fname)\n",
    "print(sub_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-15T11:17:05.906378Z",
     "start_time": "2019-08-15T11:17:04.743742Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:22:59.667874Z",
     "start_time": "2019-08-21T02:22:56.166810Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14870"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference\n",
    "\n",
    "Make sure `tranforms` are activated to test set otherwise TTA > 1 will be as TTA =1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:23:02.472345Z",
     "start_time": "2019-08-21T02:22:59.671773Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_fname = Path('test.npz')\n",
    "try:\n",
    "    npzfile  = np.load(fname_ext(test_fname, ext))\n",
    "    xt_xyz   = npzfile['x_xyz']\n",
    "    xt_type  = npzfile['x_type']\n",
    "    xt_ext   = npzfile['x_ext']\n",
    "    xt_atom  = npzfile['x_atom']\n",
    "    mt = npzfile['m']\n",
    "    xt_ids = npzfile['x_ids']\n",
    "except:\n",
    "    xt_xyz,xt_type,xt_ext,xt_atom,mt,xt_ids = \\\n",
    "        preprocess(test_fname.with_suffix('.csv'), type_index=types,ext=ext)\n",
    "    np.savez(fname_ext('_'+test_fname, ext), \n",
    "             x_xyz  = xt_xyz,\n",
    "             x_type = xt_type,\n",
    "             x_ext  = xt_ext,\n",
    "             x_atom = xt_atom,\n",
    "             m=mt,\n",
    "             x_ids=xt_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:23:02.480589Z",
     "start_time": "2019-08-21T02:23:02.474709Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    try:\n",
    "        xt_coulombmat = load_fn(f'xt_coulombmat32{ext}.npy')\n",
    "    except:\n",
    "        xt_coulombmat = np.load(f'xt_coulombmat{ext}.npy', allow_pickle=True)\n",
    "        xt_coulombmat = np.array(xt_coulombmat.tolist()).astype(np.float32)\n",
    "        np.save(f'xt_coulombmat32{ext}.npy', xt_coulombmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:23:02.487954Z",
     "start_time": "2019-08-21T02:23:02.482416Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xt_qm9_mulliken = load_fn(f'xt_qm9_mulliken{ext}.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:23:02.493831Z",
     "start_time": "2019-08-21T02:23:02.490149Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xt_qm9_mulliken_ext.npy'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f'xt_qm9_mulliken{ext}.npy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:23:02.501444Z",
     "start_time": "2019-08-21T02:23:02.495993Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(756113, 3, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 29),\n",
       " (756113,)]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.shape for v in [xt_xyz,xt_type,xt_ext,xt_atom, xt_qm9_mulliken,xt_ids, mt]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.data.add_test(ItemList(items=(MoleculeItem(i,*v) for i,v in \n",
    "                              enumerate(zip(xt_xyz,xt_type,xt_ext,xt_atom,xt_qm9_mulliken,xt_coulombmat)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:23:50.865687Z",
     "start_time": "2019-08-21T02:23:46.400798Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "del xt_xyz,xt_type,xt_ext,xt_atom, xt_qm9_mulliken, mt\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TTA_N = 1\n",
    "learner.data.test_ds.tfms = tta_tfms if TTA_N > 1 else tfms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:33:49.692070Z",
     "start_time": "2019-08-21T02:23:50.867913Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>scalar_coupling_constant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4658147</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4658148</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4658149</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4658150</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4658151</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  scalar_coupling_constant\n",
       "0  4658147                       0.0\n",
       "1  4658148                       0.0\n",
       "2  4658149                       0.0\n",
       "3  4658150                       0.0\n",
       "4  4658151                       0.0"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = defaultdict(int)\n",
    "xt_ids_not_extended = (xt_ids!=0) & (xt_ids<=7163688) # TODO\n",
    "ids = xt_ids[xt_ids_not_extended]\n",
    "\n",
    "mb = master_bar(range(TTA_N))\n",
    "for tta in mb:\n",
    "    test_preds = np.zeros((0, 29), dtype=np.float32)\n",
    "\n",
    "    for batch_idx, batch in progress_bar(\n",
    "        enumerate(learner.dl(DatasetType.Test)), total=len(learner.dl(DatasetType.Test)), parent=mb):\n",
    "        types_, _, preds_,_,_,_ = learner.pred_batch(ds_type=DatasetType.Test, batch=batch)\n",
    "        preds_ = preds_.sum(dim=1)\n",
    "        preds_[types_.squeeze(1)!=type_to_tune] = 0.\n",
    "        test_preds = np.concatenate([test_preds, preds_.data.cpu().numpy()], axis = 0)\n",
    "\n",
    "    preds = test_preds[xt_ids_not_extended]\n",
    "    for k in range(len(ids)):\n",
    "        sub[int(ids[k])] += preds[k]\n",
    "    \n",
    "for k in range(len(ids)):\n",
    "    sub[int(ids[k])] = sub[int(ids[k])]/TTA_N\n",
    "\n",
    "sub_df = pd.DataFrame(sub.items(), columns=['id', 'scalar_coupling_constant'])\n",
    "sub_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_df.to_csv(sub_fname, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'loss-0.5834val-3.0490_lmae2-2.1933'"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'loss-0.5834val-3.0490_lmae2-2.1933_val'"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.data.valid_dl = data.valid_dl.new(shuffle=False)\n",
    "TTA_N =1                                          \n",
    "sub = defaultdict(int)\n",
    "targets = defaultdict(int)\n",
    "targets_types = defaultdict(int)\n",
    "\n",
    "mb = master_bar(range(TTA_N))\n",
    "for tta in mb:\n",
    "    test_preds = np.zeros((0, 29), dtype=np.float32)\n",
    "    test_exts = np.zeros((0, 29), dtype=np.float32)\n",
    "    test_types = np.zeros((0, 29), dtype=np.float32)\n",
    "    test_targets = np.zeros((0, 29), dtype=np.float32)\n",
    "\n",
    "    for batch_idx, batch in progress_bar(\n",
    "        enumerate(learner.dl(DatasetType.Valid)), total=len(learner.dl(DatasetType.Valid)), parent=mb):\n",
    "\n",
    "        types_, ext_, preds_,_,_,_ = learner.pred_batch(ds_type=DatasetType.Valid, batch=batch)\n",
    "        preds_ = preds_.sum(dim=1)\n",
    "        preds_[types_.squeeze(1)!=type_to_tune] = 0.\n",
    "\n",
    "        test_preds = np.concatenate([test_preds, preds_.data.cpu().numpy()], axis = 0)\n",
    "\n",
    "        targets_ = batch[1][0].sum(dim=1)\n",
    "        test_targets = np.concatenate([test_targets, targets_.data.cpu().numpy()], axis = 0)\n",
    "        \n",
    "        test_types = np.concatenate([test_types, types_.data.cpu().numpy().squeeze(1)], axis = 0)\n",
    "        test_exts = np.concatenate([test_exts, ext_.data.cpu().numpy().squeeze(1)], axis = 0)\n",
    "    \n",
    "    test_preds = test_preds.flatten()\n",
    "    test_types = test_types.flatten()\n",
    "    test_targets = test_targets.flatten()\n",
    "    test_exts = test_exts.flatten()\n",
    "    \n",
    "    mask = (test_types != -1) & (test_exts == 0) #.squeeze(1)\n",
    "    preds = test_preds[mask]\n",
    "    test_targets = test_targets[mask]\n",
    "    test_types = test_types[mask]\n",
    "    for k in range(len(preds)):\n",
    "        sub[k] += preds[k]\n",
    "        targets[k] = test_targets[k]\n",
    "        targets_types[k] = test_types[k]\n",
    "    \n",
    "for k in range(len(sub.keys())):\n",
    "    sub[k] = sub[k]/TTA_N\n",
    "\n",
    "sub_df = pd.DataFrame(sub.items(), columns=['id', 'scalar_coupling_constant'])\n",
    "sub_df.head()\n",
    "sub_df.to_csv(sub_fname + '_val', index=False)\n",
    "sub_fname + '_val'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submit to Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:33:49.698278Z",
     "start_time": "2019-08-21T02:33:49.694339Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#sub_fname = 'loss-4.9044val-2.5880'\n",
    "#sub_fname='loss-4.9516val-2.8229'\n",
    "sub_fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:34:01.489116Z",
     "start_time": "2019-08-21T02:33:49.700328Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_df.to_csv(sub_fname, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:34:01.493376Z",
     "start_time": "2019-08-21T02:34:01.490900Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comp = 'champs-scalar-coupling'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:34:01.503188Z",
     "start_time": "2019-08-21T02:34:01.495166Z"
    },
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sub_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T02:34:01.507189Z",
     "start_time": "2019-08-21T02:34:01.504925Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ext=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:20:28.337421Z",
     "start_time": "2019-08-21T03:20:22.710973Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!kaggle competitions submit -c {comp} -f {sub_fname} -m 'knockout no QM9 no ext no max_atoms tta {TTA_N} {ext} transformer + attn no scale + 1024'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:21:29.884652Z",
     "start_time": "2019-08-21T03:20:28.342603Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "time.sleep(60)\n",
    "!kaggle competitions submissions -c {comp} -v > submissions-{comp}.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:21:29.911396Z",
     "start_time": "2019-08-21T03:21:29.890254Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submissions = pd.read_csv(f'submissions-{comp}.csv')\n",
    "submissions.iloc[0].publicScore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# L2 data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:09.681791Z",
     "start_time": "2019-08-21T03:22:20.434837Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.data.valid_dl = data.valid_dl.new(shuffle=False)\n",
    "TTA_N =1                                          \n",
    "sub = defaultdict(int)\n",
    "targets = defaultdict(int)\n",
    "targets_types = defaultdict(int)\n",
    "\n",
    "mb = master_bar(range(TTA_N))\n",
    "for tta in mb:\n",
    "    test_preds = np.zeros((0, 29), dtype=np.float32)\n",
    "    test_exts = np.zeros((0, 29), dtype=np.float32)\n",
    "    test_types = np.zeros((0, 29), dtype=np.float32)\n",
    "    test_targets = np.zeros((0, 29), dtype=np.float32)\n",
    "\n",
    "    for batch_idx, batch in progress_bar(\n",
    "        enumerate(learner.dl(DatasetType.Valid)), total=len(learner.dl(DatasetType.Valid)), parent=mb):\n",
    "\n",
    "        types_, ext_, preds_,_,_,_ = learner.pred_batch(ds_type=DatasetType.Valid, batch=batch)\n",
    "        preds_ = preds_.sum(dim=1)\n",
    "        test_preds = np.concatenate([test_preds, preds_.data.cpu().numpy()], axis = 0)\n",
    "\n",
    "        targets_ = batch[1][0].sum(dim=1)\n",
    "        test_targets = np.concatenate([test_targets, targets_.data.cpu().numpy()], axis = 0)\n",
    "        \n",
    "        test_types = np.concatenate([test_types, types_.data.cpu().numpy().squeeze(1)], axis = 0)\n",
    "        test_exts = np.concatenate([test_exts, ext_.data.cpu().numpy().squeeze(1)], axis = 0)\n",
    "    \n",
    "    test_preds = test_preds.flatten()\n",
    "    test_types = test_types.flatten()\n",
    "    test_targets = test_targets.flatten()\n",
    "    test_exts = test_exts.flatten()\n",
    "    \n",
    "    mask = (test_types != -1) & (test_exts == 0) #.squeeze(1)\n",
    "    preds = test_preds[mask]\n",
    "    test_targets = test_targets[mask]\n",
    "    test_types = test_types[mask]\n",
    "    for k in range(len(preds)):\n",
    "        sub[k] += preds[k]\n",
    "        targets[k] = test_targets[k]\n",
    "        targets_types[k] = test_types[k]\n",
    "    \n",
    "for k in range(len(sub.keys())):\n",
    "    sub[k] = sub[k]/TTA_N\n",
    "\n",
    "sub_df = pd.DataFrame(sub.items(), columns=['id', 'scalar_coupling_constant'])\n",
    "sub_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:10.021713Z",
     "start_time": "2019-08-21T03:24:09.684318Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_targets = pd.DataFrame(targets.items(), columns=['id', 'scalar_coupling_constant'])\n",
    "sub_targets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:10.360866Z",
     "start_time": "2019-08-21T03:24:10.024618Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_targets = pd.DataFrame(targets.items(), columns=['id', 'scalar_coupling_constant'])\n",
    "sub_targets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:10.685807Z",
     "start_time": "2019-08-21T03:24:10.363096Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_targets_types = pd.DataFrame(targets_types.items(), columns=['id', 'type'])\n",
    "sub_targets_types.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-05T21:42:00.805651Z",
     "start_time": "2019-08-05T21:41:59.063328Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_targets_types.to_csv('validation_types', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:10.695479Z",
     "start_time": "2019-08-21T03:24:10.687958Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:10.704696Z",
     "start_time": "2019-08-21T03:24:10.697572Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_targets.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-03T20:07:20.048256Z",
     "start_time": "2019-08-03T20:07:20.041023Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-03T20:09:32.533038Z",
     "start_time": "2019-08-03T20:09:30.294063Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_targets.to_csv('validation_targets', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:12.898543Z",
     "start_time": "2019-08-21T03:24:10.706332Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_df.to_csv(sub_fname + '_val', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-21T03:24:12.906168Z",
     "start_time": "2019-08-21T03:24:12.901116Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_fname + '_val'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-05T21:00:08.361139Z",
     "start_time": "2019-08-05T21:00:07.258433Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!head loss-4.9516val-3.0042_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-03T20:11:05.245077Z",
     "start_time": "2019-08-03T20:11:04.340447Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!head loss-5.7552val-2.9950_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-03T20:11:18.206277Z",
     "start_time": "2019-08-03T20:11:17.307404Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!head validation_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (root)",
   "language": "python",
   "name": "root"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}