transformer-experiments-2nd-level.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:09.549350Z",
     "start_time": "2019-07-26T05:05:08.935318Z"
    }
   },
   "outputs": [],
   "source": [
    "#%load_ext autoreload\n",
    "#%autoreload 2\n",
    "import os\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.parallel\n",
    "import torch.utils.data\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:10.060165Z",
     "start_time": "2019-07-26T05:05:10.011195Z"
    }
   },
   "outputs": [],
   "source": [
    "from torch.nn import Parameter\n",
    "\n",
    "def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict,\n",
    "                          missing_keys, unexpected_keys, error_msgs):\n",
    "    r\"\"\"Copies parameters and buffers from :attr:`state_dict` into only\n",
    "    this module, but not its descendants. This is called on every submodule\n",
    "    in :meth:`~torch.nn.Module.load_state_dict`. Metadata saved for this\n",
    "    module in input :attr:`state_dict` is provided as :attr:`local_metadata`.\n",
    "    For state dicts without metadata, :attr:`local_metadata` is empty.\n",
    "    Subclasses can achieve class-specific backward compatible loading using\n",
    "    the version number at `local_metadata.get(\"version\", None)`.\n",
    "\n",
    "    .. note::\n",
    "        :attr:`state_dict` is not the same object as the input\n",
    "        :attr:`state_dict` to :meth:`~torch.nn.Module.load_state_dict`. So\n",
    "        it can be modified.\n",
    "\n",
    "    Arguments:\n",
    "        state_dict (dict): a dict containing parameters and\n",
    "            persistent buffers.\n",
    "        prefix (str): the prefix for parameters and buffers used in this\n",
    "            module\n",
    "        local_metadata (dict): a dict containing the metadata for this module.\n",
    "            See\n",
    "        strict (bool): whether to strictly enforce that the keys in\n",
    "            :attr:`state_dict` with :attr:`prefix` match the names of\n",
    "            parameters and buffers in this module\n",
    "        missing_keys (list of str): if ``strict=True``, add missing keys to\n",
    "            this list\n",
    "        unexpected_keys (list of str): if ``strict=True``, add unexpected\n",
    "            keys to this list\n",
    "        error_msgs (list of str): error messages should be added to this\n",
    "            list, and will be reported together in\n",
    "            :meth:`~torch.nn.Module.load_state_dict`\n",
    "    \"\"\"\n",
    "    for hook in self._load_state_dict_pre_hooks.values():\n",
    "        hook(state_dict, prefix, local_metadata, strict, missing_keys,\n",
    "             unexpected_keys, error_msgs)\n",
    "\n",
    "    local_name_params = itertools.chain(self._parameters.items(),\n",
    "                                        self._buffers.items())\n",
    "    local_state = {k: v.data for k, v in local_name_params if v is not None}\n",
    "\n",
    "    for name, param in local_state.items():\n",
    "        key = prefix + name\n",
    "        if key in state_dict:\n",
    "            input_param = state_dict[key]\n",
    "\n",
    "            # Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+\n",
    "            if len(param.shape) == 0 and len(input_param.shape) == 1:\n",
    "                input_param = input_param[0]\n",
    "\n",
    "            if input_param.shape != param.shape:\n",
    "                # local shape should match the one in checkpoint\n",
    "                error_msgs.append(\n",
    "                    'size mismatch for {}: copying a param with shape {} from checkpoint, '\n",
    "                    'the shape in current model is {}.'.format(\n",
    "                        key, input_param.shape, param.shape))\n",
    "                #if not strict:\n",
    "                #    continue\n",
    "\n",
    "            if isinstance(input_param, Parameter):\n",
    "                # backwards compatibility for serialized parameters\n",
    "                input_param = input_param.data\n",
    "\n",
    "            try:\n",
    "                param.copy_(input_param)\n",
    "            except Exception:\n",
    "                error_msgs.append(\n",
    "                    'While copying the parameter named \"{}\", '\n",
    "                    'whose dimensions in the model are {} and '\n",
    "                    'whose dimensions in the checkpoint are {}.'.format(\n",
    "                        key, param.size(), input_param.size()))\n",
    "                # PG load partially\n",
    "\n",
    "                if len(input_param.size()) == 3:\n",
    "                    error_msgs.append(\n",
    "                        'Partially copying the parameter named \"{}\", '\n",
    "                        'whose dimensions in the model are {} and '\n",
    "                        'whose dimensions in the checkpoint are {}. - trying {}'\n",
    "                        .format(\n",
    "                            key, param.size(), input_param.size(),\n",
    "                            param[:input_param.size()[0], :input_param.size(\n",
    "                            )[1], :input_param.size()[2]].shape))\n",
    "                else:\n",
    "                    error_msgs.append(\n",
    "                        'Partially copying the parameter named \"{}\", '\n",
    "                        'whose dimensions in the model are {} and '\n",
    "                        'whose dimensions in the checkpoint are {}. - trying {}'\n",
    "                        .format(key, param.size(), input_param.size(),\n",
    "                                param[:input_param.size()[0]].shape))\n",
    "\n",
    "                try:\n",
    "                    new_input_param = torch.empty_like(param)\n",
    "                    new_input_param = torch.nn.init.normal_(new_input_param,\n",
    "                                                            mean=input_param.mean(),\n",
    "                                                            std=input_param.std())\n",
    "\n",
    "                    if len(input_param.size()) == 3:\n",
    "                        new_input_param[:input_param.size()[0], :input_param.\n",
    "                                        size()[1], :input_param.size(\n",
    "                                        )[2]] = input_param\n",
    "                    else:\n",
    "                        new_input_param[:input_param.size()[0]] = input_param\n",
    "                    param.copy_(new_input_param)\n",
    "                except Exception as e:\n",
    "                    assert e\n",
    "                    error_msgs.append(\n",
    "                        'Failed to load weights partially {}'.format(e))\n",
    "        elif strict:\n",
    "            missing_keys.append(key)\n",
    "\n",
    "    if strict:\n",
    "        for key in state_dict.keys():\n",
    "            if key.startswith(prefix):\n",
    "                input_name = key[len(prefix):]\n",
    "                input_name = input_name.split(\n",
    "                    '.', 1)[0]  # get the name of param/buffer/child\n",
    "                if input_name not in self._modules and input_name not in local_state:\n",
    "                    unexpected_keys.append(key)\n",
    "\n",
    "def load_state_dict(self, state_dict, strict=True):\n",
    "    r\"\"\"Copies parameters and buffers from :attr:`state_dict` into\n",
    "    this module and its descendants. If :attr:`strict` is ``True``, then\n",
    "    the keys of :attr:`state_dict` must exactly match the keys returned\n",
    "    by this module's :meth:`~torch.nn.Module.state_dict` function.\n",
    "\n",
    "    Arguments:\n",
    "        state_dict (dict): a dict containing parameters and\n",
    "            persistent buffers.\n",
    "        strict (bool, optional): whether to strictly enforce that the keys\n",
    "            in :attr:`state_dict` match the keys returned by this module's\n",
    "            :meth:`~torch.nn.Module.state_dict` function. Default: ``True``\n",
    "\n",
    "    Returns:\n",
    "        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:\n",
    "            * **missing_keys** is a list of str containing the missing keys\n",
    "            * **unexpected_keys** is a list of str containing the unexpected keys\n",
    "    \"\"\"\n",
    "    missing_keys = []\n",
    "    unexpected_keys = []\n",
    "    error_msgs = []\n",
    "\n",
    "    # copy state_dict so _load_from_state_dict can modify it\n",
    "    metadata = getattr(state_dict, '_metadata', None)\n",
    "    state_dict = state_dict.copy()\n",
    "    if metadata is not None:\n",
    "        state_dict._metadata = metadata\n",
    "\n",
    "    def load(module, prefix=''):\n",
    "        local_metadata = {} if metadata is None else metadata.get(\n",
    "            prefix[:-1], {})\n",
    "        module._load_from_state_dict(state_dict, prefix, local_metadata, True,\n",
    "                                     missing_keys, unexpected_keys, error_msgs)\n",
    "        for name, child in module._modules.items():\n",
    "            if child is not None:\n",
    "                load(child, prefix + name + '.')\n",
    "                \n",
    "    load(self)\n",
    "\n",
    "    if strict:\n",
    "        if len(unexpected_keys) > 0:\n",
    "            error_msgs.insert(\n",
    "                0, 'Unexpected key(s) in state_dict: {}. '.format(', '.join(\n",
    "                    '\"{}\"'.format(k) for k in unexpected_keys)))\n",
    "        if len(missing_keys) > 0:\n",
    "            error_msgs.insert(\n",
    "                0, 'Missing key(s) in state_dict: {}. '.format(', '.join(\n",
    "                    '\"{}\"'.format(k) for k in missing_keys)))\n",
    "\n",
    "    if strict and len(error_msgs) > 0:\n",
    "        raise RuntimeError(\n",
    "            'Error(s) in loading state_dict for {}:\\n\\t{}'.format(\n",
    "                self.__class__.__name__, \"\\n\\t\".join(error_msgs)))\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:10.649625Z",
     "start_time": "2019-07-26T05:05:10.645674Z"
    }
   },
   "outputs": [],
   "source": [
    "#nn.Module._load_from_state_dict = _load_from_state_dict\n",
    "#nn.Module.load_state_dict = load_state_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:12.210458Z",
     "start_time": "2019-07-26T05:05:11.247157Z"
    }
   },
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.basic_train import *\n",
    "from fastai.basic_data import *\n",
    "from fastai.data_block import *\n",
    "from fastai.torch_core import *\n",
    "from fastai.train import *\n",
    "from fastai.callback import *\n",
    "from fastai.callbacks import *\n",
    "from fastai.distributed import *\n",
    "from fastai.layers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:12.298113Z",
     "start_time": "2019-07-26T05:05:12.248272Z"
    }
   },
   "outputs": [],
   "source": [
    "fname_ext = lambda fname,ext: f'{str(fname)[:-4]}{ext}{str(fname)[-4:]}'\n",
    "\n",
    "def preprocess(fname, type_index=None, ext=''):\n",
    "    t  = pd.read_csv(fname_ext(fname,ext))\n",
    "    s  = pd.read_csv('structures.csv')\n",
    "    \n",
    "    has_y = 'scalar_coupling_constant' in t.columns\n",
    "\n",
    "    if has_y:\n",
    "        # atom-atom level\n",
    "        # molecule_name,atom_index_0,atom_index_1,type,fc,sd,pso,dso\n",
    "        scalar_couplings = pd.read_csv(f'scalar_coupling_contributions{ext}.csv') # fc,sd,pso,dso\n",
    "\n",
    "        # atom level\n",
    "        # molecule_name,atom_index,XX,YX,ZX,XY,YY,ZY,XZ,YZ,ZZ\n",
    "        magnetic_shielding = pd.read_csv('magnetic_shielding_tensors.csv')\n",
    "        # molecule_name,atom_index,mulliken_charge\n",
    "        mulliken_charges = pd.read_csv('mulliken_charges.csv')\n",
    "\n",
    "        # molecule level\n",
    "        # molecule_name,X,Y,Z\n",
    "        dipole_moments = pd.read_csv('dipole_moments.csv')\n",
    "        # molecule_name,potential_energy\n",
    "        potential_energy = pd.read_csv('potential_energy.csv')\n",
    "\n",
    "    t['molecule_index'] = pd.factorize(t['molecule_name'])[0] + t['id'].min()\n",
    "    # make sure we use the same indexes in train/test (test needs to provide type_index)\n",
    "    if type_index is not None:\n",
    "        t['type_idx'] = t['type'].apply(lambda x: type_index.index(x) ) # pd.factorize(pd.concat([pd.Series(type_index),t['type']]))[0][len(type_index):]\n",
    "    else:\n",
    "        t['type_idx'] = pd.factorize(t['type'])[0]\n",
    "\n",
    "    s['atom_idx'] = s['atom'].apply(lambda x: atoms.index(x) )\n",
    "\n",
    "    max_items = len(t.groupby(['molecule_name', 'atom_index_0']))# if has_y else 422550\n",
    "    max_atoms = int(s.atom_index.max() + 1)\n",
    "\n",
    "    if has_y:\n",
    "        contributions = ['fc','sd','pso','dso']\n",
    "        magnetic_tensors = ['XX','YX','ZX','XY','YY','ZY','XZ','YZ','ZZ']\n",
    "        XYZ = ['X','Y','Z']\n",
    "    xyz = ['x', 'y', 'z']\n",
    "    \n",
    "    x_xyz   = np.zeros((max_items,len(xyz),  max_atoms), dtype=np.float32)\n",
    "    x_type  = np.zeros((max_items,1,         max_atoms), dtype=np.int8)\n",
    "    x_ext   = np.zeros((max_items,1,         max_atoms), dtype=np.bool_)\n",
    "    x_atom  = np.empty((max_items,1,         max_atoms), dtype=np.int8)\n",
    "    x_atom[:] = -1\n",
    "\n",
    "    if has_y:\n",
    "        y_scalar   = np.zeros((max_items,len(contributions)   ,max_atoms), dtype=np.float32)\n",
    "        y_magnetic = np.zeros((max_items,len(magnetic_tensors),max_atoms), dtype=np.float32)\n",
    "        y_mulliken = np.zeros((max_items,1                    ,max_atoms), dtype=np.float32)\n",
    "\n",
    "        y_dipole   = np.zeros((max_items,len(XYZ)), dtype=np.float32)\n",
    "        y_potential= np.zeros((max_items,1       ), dtype=np.float32)\n",
    "\n",
    "        y_magnetic[...] = np.nan\n",
    "        y_mulliken[...] = np.nan\n",
    "    else:\n",
    "        xt_ids = np.zeros((max_items, max_atoms), dtype=np.int32)\n",
    "\n",
    "    m = np.zeros((max_items,), dtype=np.int32)\n",
    "    i = j = 0\n",
    "    \n",
    "    for (m_name, m_index) ,m_group in tqdm(t.groupby(['molecule_name', 'molecule_index'])):\n",
    "        ss = s[s.molecule_name==m_name]\n",
    "        n_atoms = len(ss)\n",
    "        if has_y:\n",
    "            magnetic = magnetic_shielding[\n",
    "                    (magnetic_shielding['molecule_name']==m_name)][magnetic_tensors].values.T\n",
    "\n",
    "            mulliken = mulliken_charges[\n",
    "                    (mulliken_charges['molecule_name']==m_name)]['mulliken_charge'].values.T\n",
    "\n",
    "            scs = scalar_couplings[scalar_couplings['molecule_name']==m_name]\n",
    "            \n",
    "            y_dipole[j,:]= dipole_moments[dipole_moments['molecule_name']==m_name][XYZ].values\n",
    "            y_potential[j,:]=potential_energy[\n",
    "                potential_energy['molecule_name']==m_name]['potential_energy'].values\n",
    "        \n",
    "        for a_name,a_group in m_group.groupby('atom_index_0'):\n",
    "            \n",
    "            ref_a = ss[ss['atom_index']==a_name]\n",
    "            \n",
    "            x_xyz[i] = 0.\n",
    "            x_type[i] = -1\n",
    "            x_ext[i] =  True\n",
    "            \n",
    "            x_xyz[i,:,:n_atoms] = (ss[xyz].values-ref_a[xyz].values).T  # xyz \n",
    "            x_type[i,0,a_group['atom_index_1']] = a_group['type_idx']  # type \n",
    "            x_ext[i,0,a_group['atom_index_1']] = a_group['ext']  # ext \n",
    "            x_atom[i,:,:n_atoms] = ss['atom_idx'].T                \n",
    "\n",
    "            if has_y:\n",
    "                y_scalar[i,:,a_group['atom_index_1']] = scs[scs['atom_index_0']==a_name][contributions]\n",
    "                y_magnetic[i,:,:n_atoms] = magnetic\n",
    "                y_mulliken[i,:,:n_atoms] = mulliken\n",
    "            else:\n",
    "                xt_ids[i,a_group['atom_index_1']] = a_group['id']  \n",
    "\n",
    "            m[i] = m_index\n",
    "            i+=1\n",
    "        j += 1\n",
    "    assert i == max_items\n",
    "    print(i,max_items)\n",
    "    if has_y:\n",
    "        return x_xyz,x_type,x_ext,x_atom, m, y_scalar, y_magnetic, y_mulliken, y_dipole, y_potential\n",
    "    else:\n",
    "        return x_xyz,x_type,x_ext,x_atom, m, xt_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define where you want to use original training set '' or extended ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:13.555649Z",
     "start_time": "2019-07-26T05:05:13.551653Z"
    }
   },
   "outputs": [],
   "source": [
    "ext = '_ext' # or ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load preprocessed or preprocess and save for later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:19.743533Z",
     "start_time": "2019-07-26T05:05:14.699472Z"
    }
   },
   "outputs": [],
   "source": [
    "train_fname = Path('train.npz')\n",
    "types = ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN'] \n",
    "atoms = 'CFHNO'\n",
    "\n",
    "try:\n",
    "    npzfile = np.load(fname_ext(train_fname, ext))\n",
    "    x_xyz   = npzfile['x_xyz']\n",
    "    x_type  = npzfile['x_type']\n",
    "    x_ext   = npzfile['x_ext']\n",
    "    x_atom  = npzfile['x_atom']\n",
    "\n",
    "    y_scalar    = npzfile['y_scalar']\n",
    "    y_magnetic  = npzfile['y_magnetic']\n",
    "    y_mulliken  = npzfile['y_mulliken']\n",
    "    y_dipole    = npzfile['y_dipole']\n",
    "    y_potential = npzfile['y_potential']\n",
    "    m = npzfile['m']\n",
    "    max_items, max_atoms = x_xyz.shape[0], x_xyz.shape[-1]\n",
    "except:\n",
    "    x_xyz,x_type,x_ext,x_atom, m, y_scalar, y_magnetic, y_mulliken, y_dipole, y_potential = \\\n",
    "        preprocess(train_fname.with_suffix('.csv'), type_index=types, ext=ext)\n",
    "    np.savez(fname_ext(train_fname, ext), \n",
    "             x_xyz=x_xyz,\n",
    "             x_type=x_type,\n",
    "             x_ext=x_ext,\n",
    "             x_atom=x_atom,\n",
    "             y_scalar=y_scalar,\n",
    "             y_magnetic=y_magnetic,\n",
    "             y_mulliken=y_mulliken,\n",
    "             y_dipole=y_dipole,\n",
    "             y_potential=y_potential,\n",
    "             m=m)\n",
    "n_types = int(x_type[~np.isnan(x_type)].max() + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:19.753758Z",
     "start_time": "2019-07-26T05:05:19.746125Z"
    }
   },
   "outputs": [],
   "source": [
    "use_memmap = True\n",
    "try:\n",
    "    load_fn = np.load if not use_memmap else partial(np.lib.format.open_memmap, mode='r')\n",
    "    x_coulombmat = load_fn(f'x_coulombmat32{ext}.npy')\n",
    "except:\n",
    "    x_coulombmat = np.load(f'x_coulombmat{ext}.npy', allow_pickle=True)\n",
    "    x_coulombmat = np.array(x_coulombmat.tolist()).astype(np.float32)\n",
    "    np.save(f'x_coulombmat32{ext}.npy', x_coulombmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:19.760746Z",
     "start_time": "2019-07-26T05:05:19.756800Z"
    }
   },
   "outputs": [],
   "source": [
    "x_qm9_mulliken = load_fn(f'x_qm9_mulliken{ext}.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:19.772530Z",
     "start_time": "2019-07-26T05:05:19.763433Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1405126, 3, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 4, 29),\n",
       " (1405126, 9, 29),\n",
       " (1405126, 1, 29),\n",
       " (1405126, 3),\n",
       " (1405126, 1),\n",
       " (1405126,)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.shape for v in [x_xyz,x_type,x_ext,x_atom,x_qm9_mulliken, \n",
    "                   y_scalar, y_magnetic, y_mulliken, y_dipole, y_potential, m]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:21.056315Z",
     "start_time": "2019-07-26T05:05:19.774767Z"
    }
   },
   "outputs": [],
   "source": [
    "x_xyz_mean, x_xyz_std = Tensor(x_xyz.mean(axis=(0,2),keepdims=True)), Tensor(x_xyz.std(axis=(0,2),keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:21.157220Z",
     "start_time": "2019-07-26T05:05:21.058305Z"
    }
   },
   "outputs": [],
   "source": [
    "x_qm9_mulliken_mean = Tensor(x_qm9_mulliken.mean(axis=(0,2),keepdims=True))\n",
    "x_qm9_mulliken_std  = Tensor(x_qm9_mulliken.std( axis=(0,2),keepdims=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:21.163088Z",
     "start_time": "2019-07-26T05:05:21.159154Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3, 1])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_xyz_std.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fastai classes (this should should be done into its own `application` but who has time?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:21.189374Z",
     "start_time": "2019-07-26T05:05:21.166096Z"
    }
   },
   "outputs": [],
   "source": [
    "class MoleculeItem(ItemBase):\n",
    "    def __init__(self,i,xyz,type,ext,atom,qm9_mulliken,coulomb): \n",
    "        self.i,self.xyz,self.type,self.ext, self.atom,self.qm9_mulliken,self.coulomb = \\\n",
    "            i,xyz,type,ext,atom,qm9_mulliken,coulomb\n",
    "        self.data = [Tensor(xyz), LongTensor((type)), \n",
    "                     Tensor(ext), LongTensor((atom)),Tensor(qm9_mulliken), Tensor(coulomb)]\n",
    "    def __str__(self):\n",
    "        # TODO: count n_atoms correctly. \n",
    "        n_atoms = np.count_nonzero(np.sum(np.absolute(self.xyz), axis=0))+1\n",
    "        n_couplings = np.sum((self.type!=-1))\n",
    "        return f'{self.i} {n_atoms} atoms {n_couplings} couplings'\n",
    "    \n",
    "    def apply_tfms(self, tfms:Collection, **kwargs):\n",
    "        x = self.clone()\n",
    "        for t in tfms:\n",
    "            if t: x.data = t(x.data)\n",
    "        return x\n",
    "    \n",
    "    def clone(self):\n",
    "        return self.__class__(self.i,self.xyz,self.type,self.ext,self.atom,self.qm9_mulliken,self.coulomb)\n",
    "    \n",
    "class ScalarCouplingItem(ItemBase):\n",
    "    def __init__(self,scalar,magnetic,mulliken,dipole,potential,**kwargs): \n",
    "        self.scalar,self.magnetic,self.mulliken,self.dipole,self.potential = \\\n",
    "            scalar,magnetic,mulliken,dipole,potential\n",
    "        self.data = (Tensor(scalar), Tensor(magnetic), Tensor(dipole), Tensor(potential))\n",
    "    def __str__(self):\n",
    "        res, spacer, n_couplings = '', '', 0\n",
    "        for s in self.data[0].sum(dim=0):\n",
    "            if s==0.: spacer = ' * '\n",
    "            else: \n",
    "                res += f'{spacer}{s:.4f}'\n",
    "                spacer = ' '\n",
    "                n_couplings +=1\n",
    "        return f'{n_couplings}: {res}'\n",
    "    def __hash__(self): return hash(str(self))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:21.215283Z",
     "start_time": "2019-07-26T05:05:21.191652Z"
    }
   },
   "outputs": [],
   "source": [
    "class LMAEMaskedLoss(Module):\n",
    "    def __init__(self,\n",
    "                 contrib_w=0., magnetic_w=0., dipole_w=0., potential_w=0., \n",
    "                 types_w = [1]*n_types, return_all=False, proxy_log=torch.log, exclude_ext=False):\n",
    "        self.contrib_w,self.magnetic_w,self.dipole_w,self.potential_w = contrib_w,magnetic_w,dipole_w,potential_w\n",
    "        self.types_w = types_w\n",
    "        self.return_all = return_all\n",
    "        self.proxy_log = proxy_log\n",
    "        self.exclude_ext = exclude_ext\n",
    "    \n",
    "    def forward(self, input_outputs, t_scalar, t_magnetic, t_dipole, t_potential):    \n",
    "        type, ext, p_scalar, p_magnetic, p_dipole, p_potential = input_outputs\n",
    "        loss = 0.\n",
    "        n = 0\n",
    "        j_loss = [0] * n_types\n",
    "        for t in range(n_types):\n",
    "            mask = (type == t).squeeze(1) if not self.exclude_ext else ((type == t) & (ext == 0)).squeeze(1)\n",
    "            if mask.sum() > 0:\n",
    "                _output,_target = p_scalar.transpose(1,2)[mask], t_scalar.transpose(1,2)[mask] # scalars at the end\n",
    "                # LMAE scalar\n",
    "                s_loss = self.proxy_log((_output.sum(dim=-1) - _target.sum(dim=-1)).abs().mean()+1e-9)\n",
    "                loss += self.types_w[t] * s_loss\n",
    "                j_loss[t] += s_loss\n",
    "                # LMAE scalar contributions\n",
    "                for i_contrib in range(_output.shape[-1]):\n",
    "                    loss += self.contrib_w * \\\n",
    "                        self.proxy_log((_output[...,i_contrib] - _target[...,i_contrib]).abs().mean()+1e-9)\n",
    "                n+=1\n",
    "        loss /= n\n",
    "        \n",
    "        if self.magnetic_w > 0:\n",
    "            mask = ~torch.isnan(t_magnetic)\n",
    "            loss += self.magnetic_w * MSELossFlat()(p_magnetic[mask], t_magnetic[mask])\n",
    "            \n",
    "        if self.dipole_w    > 0: loss += self.dipole_w    * MSELossFlat()(p_dipole,    t_dipole)\n",
    "        if self.potential_w > 0: loss += self.potential_w * MSELossFlat()(p_potential, t_potential)\n",
    "\n",
    "        return loss if not self.return_all else (loss, *j_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:21.231156Z",
     "start_time": "2019-07-26T05:05:21.217051Z"
    }
   },
   "outputs": [],
   "source": [
    "class ScalarCouplingList(ItemList):\n",
    "    def __init__(self, items:Iterator, **kwargs):\n",
    "        super().__init__(items, **kwargs)\n",
    "        self.loss_func = LMAEMaskedLoss\n",
    "\n",
    "    def get(self, i):\n",
    "        o = super().get(i)\n",
    "        return ScalarCouplingItem(*o)\n",
    "\n",
    "    def reconstruct(self,t): return 0; # TODO for viz !!!! ScalarCouplingItem(t.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Quaterions allow us to rotate 3d points randoming with a nice uniform distribution of 3 numbers hece we use them, however it's still to be seen if are useful here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:05:23.809400Z",
     "start_time": "2019-07-26T05:05:23.775606Z"
    }
   },
   "outputs": [],
   "source": [
    "#https://github.com/facebookresearch/QuaterNet/blob/master/common/quaternion.py\n",
    "def qrot(q, v):\n",
    "    \"\"\"\n",
    "    Rotate vector(s) v about the rotation described by quaternion(s) q.\n",
    "    Expects a tensor of shape (*, 4) for q and a tensor of shape (*, 3) for v,\n",
    "    where * denotes any number of dimensions.\n",
    "    Returns a tensor of shape (*, 3).\n",
    "    \"\"\"\n",
    "    assert q.shape[-1] == 4\n",
    "    assert v.shape[-1] == 3\n",
    "    assert q.shape[:-1] == v.shape[:-1]\n",
    "    \n",
    "    original_shape = list(v.shape)\n",
    "    q = q.view(-1, 4)\n",
    "    v = v.view(-1, 3)\n",
    "    \n",
    "    qvec = q[:, 1:]\n",
    "    uv = torch.cross(qvec, v, dim=1)\n",
    "    uuv = torch.cross(qvec, uv, dim=1)\n",
    "    return (v + 2 * (q[:, :1] * uv + uuv)).view(original_shape)\n",
    "\n",
    "def random_rotation(data):\n",
    "    x_xyz = data[0].transpose(0,1)\n",
    "    r = torch.rand(3)\n",
    "    sq1_v1,sqv1,v2_2pi,v3_2pi = torch.sqrt(1-r[:1]),torch.sqrt(r[:1]),2*math.pi*r[1:2],2*math.pi*r[2:3]\n",
    "    q = torch.cat([sq1_v1*torch.sin(v2_2pi), sq1_v1*torch.cos(v2_2pi), \n",
    "                   sqv1  *torch.sin(v3_2pi), sqv1  *torch.cos(v3_2pi)], dim=0).unsqueeze(0)\n",
    "    x_xyz = qrot(q.expand(x_xyz.shape[0],-1), x_xyz).squeeze(0).transpose(0,1)\n",
    "    return (x_xyz, *data[1:])\n",
    "\n",
    "def normalize(data):\n",
    "    sq = False\n",
    "    if data[0].ndim < 3:\n",
    "        data[0].unsqueeze_(0)\n",
    "        data[4].unsqueeze_(0)\n",
    "        sq = True\n",
    "    x_xyz      = (data[0] - x_xyz_mean)          / x_xyz_std\n",
    "    x_mulliken = (data[4] - x_qm9_mulliken_mean) / x_qm9_mulliken_std\n",
    "    if sq:\n",
    "        x_xyz.squeeze_(0)\n",
    "        x_mulliken.squeeze_(0)\n",
    "    return (x_xyz, data[1],data[2],data[3],x_mulliken,data[5])\n",
    "\n",
    "def canonize(data):\n",
    "    xyz,type,ext,atom,mulliken,coulomb = data\n",
    "    mask = (atom == -1).squeeze(0)\n",
    "    mask_atoms = ~mask.unsqueeze(0)\n",
    "    n_atoms = mask_atoms.sum()\n",
    "    i = torch.nonzero(type.squeeze(0) == -1)[0] # pick first one w/o j-coupling\n",
    "    xyz[:,mask], type[:,mask],ext[:,mask],atom[:,mask],mulliken[:,mask] = \\\n",
    "        0,-1,1,-1,0\n",
    "#        xyz[:,i],type[:,i],   ext[:,i],   atom[:,i],   mulliken[:,i]\n",
    "    return (xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build `data` bunch etc. for fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:06:54.295580Z",
     "start_time": "2019-07-26T05:05:25.746091Z"
    }
   },
   "outputs": [],
   "source": [
    "data = ItemList(items=(MoleculeItem(i,*v) for i,v in \n",
    "                       enumerate(zip(x_xyz,x_type,x_ext,x_atom,x_qm9_mulliken,x_coulombmat))),\n",
    "                label_cls=ScalarCouplingItem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:06:54.790843Z",
     "start_time": "2019-07-26T05:06:54.297937Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "idx_train_split, idx_valid_split = train_test_split(range(m.max()+1), test_size=0.1, random_state=13)\n",
    "idx_valid_split = np.argwhere(np.isin(m, idx_valid_split)).flatten()\n",
    "idx_train_split = np.argwhere(np.isin(m, idx_train_split)).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:06:55.038942Z",
     "start_time": "2019-07-26T05:06:54.794461Z"
    },
    "pixiedust": {
     "displayParams": {}
    }
   },
   "outputs": [],
   "source": [
    "data = data.split_by_idx(idx_train_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:06:59.183594Z",
     "start_time": "2019-07-26T05:06:55.041194Z"
    }
   },
   "outputs": [],
   "source": [
    "data = data.label_from_func(\n",
    "    func=lambda o: (y_scalar[o.i], y_magnetic[o.i], y_mulliken[o.i], y_dipole[o.i], y_potential[o.i]),\n",
    "    label_cls=ScalarCouplingList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T21:55:21.645222Z",
     "start_time": "2019-07-25T21:55:21.640287Z"
    }
   },
   "outputs": [],
   "source": [
    "if False:\n",
    "    try:\n",
    "        xt_coulombmat = load_fn(f'xt_coulombmat32{ext}.npy')\n",
    "    except:\n",
    "        xt_coulombmat = np.load(f'xt_coulombmat{ext}.npy', allow_pickle=True)\n",
    "        xt_coulombmat = np.array(xt_coulombmat.tolist()).astype(np.float32)\n",
    "        np.save(f'xt_coulombmat32{ext}.npy', xt_coulombmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:06:59.191048Z",
     "start_time": "2019-07-26T05:06:59.185743Z"
    }
   },
   "outputs": [],
   "source": [
    "tfms = [normalize, canonize]\n",
    "tta_tfms = list(tfms)\n",
    "#tta_tfms.insert(0,random_rotation)\n",
    "data = data.transform((tta_tfms, tfms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:06:59.227649Z",
     "start_time": "2019-07-26T05:06:59.193113Z"
    }
   },
   "outputs": [],
   "source": [
    "data=data.databunch(num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:07:00.343940Z",
     "start_time": "2019-07-26T05:06:59.229765Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LabelList (0 items)\n",
       "x: ItemList\n",
       "\n",
       "y: ScalarCouplingList\n",
       "\n",
       "Path: ."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.valid_ds.filter_by_func(lambda item, _: True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "Whole model here, self-contained (needs some cleanup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T21:55:22.864678Z",
     "start_time": "2019-07-25T21:55:22.856409Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataBunch;\n",
       "\n",
       "Train: LabelList (140165 items)\n",
       "x: ItemList\n",
       "14 8 atoms 6 couplings,15 8 atoms 6 couplings,16 8 atoms 7 couplings,17 8 atoms 7 couplings,18 8 atoms 7 couplings\n",
       "y: ScalarCouplingList\n",
       "6:  * 83.5430 83.5417 83.5484 -2.3788 -2.3785 -2.3772,6:  * -2.3783 -2.3786 -2.3772 83.5418 83.5430 83.5486,7: 83.5430 -2.3783 * -11.7004 -11.6979 3.2528 13.6913 3.2521,7: 83.5417 -2.3786 -11.7004 * -11.6996 13.6924 3.2525 3.2527,7: 83.5484 -2.3772 -11.6979 -11.6996 * 3.2524 3.2524 13.6921\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (0 items)\n",
       "x: ItemList\n",
       "\n",
       "y: ScalarCouplingList\n",
       "\n",
       "Path: .;\n",
       "\n",
       "Test: None"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:07:00.363333Z",
     "start_time": "2019-07-26T05:07:00.347023Z"
    }
   },
   "outputs": [],
   "source": [
    "class LMAEMetric(LearnerCallback):\n",
    "    _order=-20 # Needs to run before the recorder\n",
    "    def __init__(self, learn, val_only=True):\n",
    "        super().__init__(learn)\n",
    "        self.val_only=val_only\n",
    "        self.metric = LMAEMaskedLoss(return_all=True, exclude_ext=True)\n",
    "\n",
    "    def on_train_begin(self, **kwargs):\n",
    "        if not self.val_only: self.learn.recorder.add_metric_names(['tLMAE'])\n",
    "        self.learn.recorder.add_metric_names(['👉🏻LMAE👈🏻'] + [f'lmae{i}' for i in range(n_types)])\n",
    "            \n",
    "    def on_batch_end(self, train, last_output, last_target, **kwargs):\n",
    "        if self.val_only and train: return \n",
    "        preds,targs = self.preds[int(train)], self.targs[int(train)] # 0 val 1 train\n",
    "        if preds is None:\n",
    "            targs, preds = listify(last_target), listify(last_output)\n",
    "            targs,preds = [t.detach() for t in targs],[t.detach() for t in preds]\n",
    "        else:\n",
    "            for i,(o,t) in enumerate(zip(last_output, last_target)):\n",
    "                preds[i] = torch.cat([preds[i], o.detach()], dim=0)\n",
    "                targs[i] = torch.cat([targs[i], t.detach()], dim=0)\n",
    "        self.preds[int(train)], self.targs[int(train)] = preds,targs\n",
    "        \n",
    "    def on_epoch_begin(self, **kwargs):\n",
    "        self.targs, self.preds = [None, None], [None, None]\n",
    "\n",
    "    def on_epoch_end(self, last_metrics, **kwargs):\n",
    "        mets = []\n",
    "        if self.preds[1]: mets.append(self.metric.forward(self.preds[1], *self.targs[1])[0]) # just tLMAE\n",
    "        if self.preds[0]: mets.extend(self.metric.forward(self.preds[0], *self.targs[0]))\n",
    "        return add_metrics(last_metrics, mets) if mets else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:07:00.432937Z",
     "start_time": "2019-07-26T05:07:00.365720Z"
    }
   },
   "outputs": [],
   "source": [
    "Activation = Enum('Activation', 'ReLU Swish GeLU')\n",
    "\n",
    "class PositionalEncoding(Module):\n",
    "    \"Implement the PE function.\"\n",
    "    def __init__(self, d_model, dropout, max_len=5000):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        \n",
    "        # Compute the positional encodings once in log space.\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        \n",
    "        position = torch.arange(0., max_len).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0., d_model, 2) * -(math.log(10000.0) / d_model))\n",
    "\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)[:, :-1]\n",
    "        pe = pe.unsqueeze(0)\n",
    "        self.register_buffer('pe', pe)\n",
    "        print(pe.size())\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.transpose(2, 1)\n",
    "        x = x + Variable(self.pe[:, :x.size(1)], \n",
    "                         requires_grad=False)\n",
    "        return self.dropout(x).transpose(2, 1)\n",
    "    \n",
    "class GeLU(Module):\n",
    "    def forward(self, x): return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))\n",
    "\n",
    "class Swish(Module):\n",
    "    def forward(self, x): return x * torch.sigmoid(x)\n",
    "\n",
    "_activ_func = {Activation.ReLU:nn.ReLU(inplace=True), Activation.GeLU:GeLU(), Activation.Swish: Swish()}\n",
    "\n",
    "def feed_forward(d_model:int, d_ff:int, ff_p:float=0., act:Activation=Activation.ReLU, double_drop:bool=True):\n",
    "    layers = [nn.Linear(d_model, d_ff), _activ_func[act]]\n",
    "    if double_drop: layers.append(nn.Dropout(ff_p))\n",
    "    return SequentialEx(*layers, nn.Linear(d_ff, d_model), nn.Dropout(ff_p), MergeLayer(), nn.LayerNorm(d_model))\n",
    "\n",
    "\n",
    "class MultiHeadAttentionOld(Module):\n",
    "    \"MutiHeadAttention.\"\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int=None, resid_p:float=0., attn_p:float=0., bias:bool=True,\n",
    "                 scale:bool=True):\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
    "        self.attention = nn.Linear(d_model, 3 * n_heads * d_head, bias=bias)\n",
    "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
    "        self.drop_att,self.drop_res = nn.Dropout(attn_p),nn.Dropout(resid_p)\n",
    "        self.ln = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs):\n",
    "        return self.ln(x + self.drop_res(self.out(self._apply_attention(x, mask=mask, **kwargs))))\n",
    "\n",
    "    def _apply_attention(self, x:Tensor, mask:Tensor=None):\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = torch.chunk(self.attention(x), 3, dim=-1)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        wq,wk,wv = wq.permute(0, 2, 1, 3),wk.permute(0, 2, 3, 1),wv.permute(0, 2, 1, 3)\n",
    "        attn_score = torch.matmul(wq, wk)\n",
    "        if self.scale: attn_score.div_(self.d_head ** 0.5)\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask,  -1.0E9).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
    "        attn_vec = torch.matmul(attn_prob, wv)\n",
    "        return attn_vec.permute(0, 2, 1, 3).contiguous().contiguous().view(bs, x_len, -1)\n",
    "\n",
    "    \n",
    "class MultiHeadAttention(Module):\n",
    "    \"MutiHeadAttention.\"\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int=None, resid_p:float=0., attn_p:float=0., bias:bool=False,\n",
    "                 scale:bool=True):\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.n_heads,self.d_head,self.scale = n_heads,d_head,scale\n",
    "\n",
    "        self.attention1 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention2 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention3 = nn.Linear(d_model, n_heads * d_head, bias=bias)\n",
    "        self.attention4 = nn.Linear(d_model, d_model, bias=bias)\n",
    "        self.out = nn.Linear(n_heads * d_head, d_model, bias=bias)\n",
    "        self.drop_att,self.drop_res = nn.Dropout(attn_p),nn.Dropout(resid_p)\n",
    "        self.ln = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs):\n",
    "        return self.ln(x + self.drop_res(self.out(self._apply_attention(x, mask=mask, **kwargs))))\n",
    "\n",
    "    def _apply_attention(self, x:Tensor, mask:Tensor=None):\n",
    "        #if mask is not None:\n",
    "        #    x= x.masked_fill(mask[:,:,0].squeeze(1).unsqueeze(-1).repeat(1, 1, x.size(2)),  0.)\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = self.attention1(x), self.attention2(x), self.attention3(x)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        wq,wk,wv = wq.permute(0, 2, 1, 3),wk.permute(0, 2, 3, 1),wv.permute(0, 2, 1, 3)\n",
    "        attn_score = torch.matmul(wq, wk)\n",
    "        if self.scale: attn_score.div_(self.d_head ** 0.5)\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask,  -1.0E9).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=-1))\n",
    "        attn_vec = torch.matmul(attn_prob, wv)\n",
    "        return self.attention4(attn_vec.permute(0, 2, 1, 3).contiguous().contiguous().view(bs, x_len, -1))\n",
    "\n",
    "    def _attention_einsum(self, x, mask=None):\n",
    "        # Permute and matmul is a little bit faster but this implementation is more readable\n",
    "        bs,x_len = x.size(0),x.size(1)\n",
    "        wq,wk,wv = torch.chunk(self.attention(x), 3, dim=-1)\n",
    "        wq,wk,wv = map(lambda x:x.view(bs, x.size(1), self.n_heads, self.d_head), (wq,wk,wv))\n",
    "        attn_score = torch.einsum('bind,bjnd->bijn', (wq, wk))\n",
    "        if self.scale: attn_score.mul_(1/(self.d_head ** 0.5))\n",
    "        if mask is not None:\n",
    "            attn_score = attn_score.float().masked_fill(mask, -float('-inf')).type_as(attn_score)\n",
    "        attn_prob = self.drop_att(F.softmax(attn_score, dim=2))\n",
    "        attn_vec = torch.einsum('bijn,bjnd->bind', (attn_prob, wv))\n",
    "        return attn_vec.contiguous().view(bs, x_len, -1)\n",
    "\n",
    "\n",
    "class DecoderLayer(Module):\n",
    "    \"Basic block of a Transformer model.\"\n",
    "    #Can't use Sequential directly cause more than one input...\n",
    "    def __init__(self, n_heads:int, d_model:int, d_head:int, d_inner:int, resid_p:float=0., attn_p:float=0., ff_p:float=0.,\n",
    "                 bias:bool=True, scale:bool=True, act:Activation=Activation.ReLU, double_drop:bool=True,\n",
    "                 attn_cls:Callable=MultiHeadAttention):\n",
    "        self.mhra = attn_cls(n_heads, d_model, d_head, resid_p=resid_p, attn_p=attn_p, bias=bias, scale=scale)\n",
    "        self.ff   = feed_forward(d_model, d_inner, ff_p=ff_p, act=act, double_drop=double_drop)\n",
    "\n",
    "    def forward(self, x:Tensor, mask:Tensor=None, **kwargs): return self.ff(self.mhra(x, mask=mask, **kwargs))\n",
    "\n",
    "class Transformer(Module):\n",
    "    \"Transformer model: https://arxiv.org/abs/1706.03762.\"\n",
    "    def __init__(self, n_layers:int, n_heads:int, d_model:int, d_head:int, d_inner:int,\n",
    "                 resid_p:float=0., attn_p:float=0., ff_p:float=0., embed_p:float=0., bias:bool=True, scale:bool=True,\n",
    "                 act:Activation=Activation.ReLU, double_drop:bool=True, attn_cls:Callable=MultiHeadAttention,\n",
    "                 learned_pos_enc:bool=True, mask:bool=True):\n",
    "        self.mask = mask\n",
    "        #self.encoder = nn.Embedding(vocab_sz, d_model)\n",
    "        #self.pos_enc = nn.Embedding(ctx_len, d_model) if learned_pos_enc else PositionalEncoding(d_model)\n",
    "        #self.pos_enc = PositionalEncoding(29, 0.1)\n",
    "        self.drop_emb = nn.Dropout(embed_p)\n",
    "        self.layers = nn.ModuleList([DecoderLayer(n_heads, d_model, d_head, d_inner, resid_p=resid_p, attn_p=attn_p,\n",
    "                      ff_p=ff_p, bias=bias, scale=scale, act=act, double_drop=double_drop,\n",
    "                      attn_cls=attn_cls) for k in range(n_layers)])\n",
    "\n",
    "    def reset(self): pass\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        #bs, x_len = x.size()\n",
    "        #pos = torch.arange(0, x_len, device=x.device, dtype=x.dtype)\n",
    "        inp = self.drop_emb(x)# + self.pos_enc(pos)[None]) #.mul_(self.d_model ** 0.5)\n",
    "        #if False:\n",
    "        #    inp += self.pos_enc(x)[None]\n",
    "        #mask = None #torch.triu(x.new_ones(x_len, x_len), diagonal=1).byte()[None,None] if self.mask else None\n",
    "        #[None,:,:None] for einsum implementation of attention\n",
    "        for layer in self.layers: inp = layer(inp, mask=mask)\n",
    "        return inp #For the LinearDecoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:07:00.465739Z",
     "start_time": "2019-07-26T05:07:00.435309Z"
    }
   },
   "outputs": [],
   "source": [
    "class AtomTransformerOld(Module):\n",
    "    def __init__(self,  n_heads,d_model, d_head=None, **kwargs):\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.transformer = Transformer(n_heads=n_heads,d_model=d_model, d_head=d_head, **kwargs)\n",
    "        \n",
    "        #self.scalar    = nn.Conv1d(d_model+ n_types + 1, 4, 1)\n",
    "        self.scalar    = nn.Conv1d(d_model, 4, 1)\n",
    "        self.magnetic  = nn.Conv1d(d_model, 9, 1)\n",
    "        self.dipole    = nn.Linear(d_model*max_atoms, 3)\n",
    "        self.potential = nn.Linear(d_model*max_atoms, 1)\n",
    "        \n",
    "        self.n_atom_embedding = d_model//2\n",
    "        self.n_type_embedding = d_model - self.n_atom_embedding - 6\n",
    "        self.type_embedding = nn.Embedding(len(types)+1,self.n_type_embedding)\n",
    "        self.atom_embedding = nn.Embedding(len(atoms)+1,self.n_atom_embedding)\n",
    "    \n",
    "        #n_pos_encoder = d_model - n_type_embedding - n_atom_embedding\n",
    "        #self.pos_encoder = nn.Sequential(\n",
    "        #    nn.Conv1d(3+1+1,n_pos_encoder, 1), nn.ReLU(), nn.BatchNorm1d(n_pos_encoder),\n",
    "        #)\n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):\n",
    "        bs, _, n_pts = xyz.shape        \n",
    "        t = self.type_embedding((type+1).squeeze(1)) #* math.sqrt(self.n_atom_embedding) #.transpose(1,2)\n",
    "        a = self.atom_embedding((atom+1).squeeze(1)) #* math.sqrt(self.n_type_embedding) #.transpose(1,2)\n",
    "        \n",
    "        x = torch.cat([xyz, mulliken, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)        \n",
    "        #x = self.pos_encoder(x).transpose(1,2)\n",
    "\n",
    "        x = torch.cat([x.transpose(1,2), t, a], dim=-1) \n",
    "\n",
    "        mask = (coulomb == 0).unsqueeze(1)\n",
    "        #mask = torch.triu(x.new_ones(max_atoms, max_atoms), diagonal=1).byte()[None,:,:,None]#[None,None] \n",
    "        #print(mask.shape, mask)\n",
    "        x = self.transformer(x, mask).transpose(1,2).contiguous()\n",
    "        \n",
    "        #t_one_hot = torch.zeros(bs,n_types+1,n_pts,device=type.device,dtype=x.dtype).scatter_(1,type+1, 1.)\n",
    "        #scalar    = self.scalar(torch.cat([x, t_one_hot], dim=1))\n",
    "        scalar    = self.scalar(x)\n",
    "        \n",
    "        magnetic  = self.magnetic(x) \n",
    "        dipole    = self.dipole(x.view(bs,-1))\n",
    "        potential = self.potential(x.view(bs,-1))\n",
    "                \n",
    "        return type,ext,scalar,magnetic,dipole,potential\n",
    "    \n",
    "    def reset(self): pass\n",
    "    \n",
    "    \n",
    "class AtomTransformer(Module):\n",
    "    def __init__(self,  n_heads,d_model, d_head=None, **kwargs):\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        d_head = ifnone(d_head, d_model//n_heads)\n",
    "        self.transformer = Transformer(n_heads=n_heads,d_model=d_model, d_head=d_head, **kwargs)\n",
    "        \n",
    "        self.scalar    = nn.Conv1d(d_model+ n_types + 1, 4, 1)\n",
    "        #self.scalar    = nn.Conv1d(d_model, 4, 1)\n",
    "        self.magnetic  = nn.Conv1d(d_model, 9, 1)\n",
    "        self.dipole    = nn.Linear(d_model*max_atoms, 3)\n",
    "        self.potential = nn.Linear(d_model*max_atoms, 1)\n",
    "        \n",
    "        self.n_atom_embedding = d_model//2\n",
    "        self.n_type_embedding = d_model - self.n_atom_embedding - 6\n",
    "        self.type_embedding = nn.Embedding(len(types)+1,self.n_type_embedding)\n",
    "        self.atom_embedding = nn.Embedding(len(atoms)+1,self.n_atom_embedding)\n",
    "    \n",
    "        #n_pos_encoder = d_model - n_type_embedding - n_atom_embedding\n",
    "        #self.pos_encoder = nn.Sequential(\n",
    "        #    nn.Conv1d(3+1+1,n_pos_encoder, 1), nn.ReLU(), nn.BatchNorm1d(n_pos_encoder),\n",
    "        #)\n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):\n",
    "        bs, _, n_pts = xyz.shape        \n",
    "        t = self.type_embedding((type+1).squeeze(1)) #* math.sqrt(self.n_atom_embedding) #.transpose(1,2)\n",
    "        a = self.atom_embedding((atom+1).squeeze(1)) #* math.sqrt(self.n_type_embedding) #.transpose(1,2)\n",
    "        \n",
    "        x = torch.cat([xyz, mulliken, ext, mask_atoms.float()], dim=1) #* math.sqrt(self.d_model)        \n",
    "        #x = self.pos_encoder(x).transpose(1,2)\n",
    "\n",
    "        x = torch.cat([x.transpose(1,2), t, a], dim=-1) \n",
    "\n",
    "        mask = (coulomb == 0).unsqueeze(1)\n",
    "        #mask = torch.triu(x.new_ones(max_atoms, max_atoms), diagonal=1).byte()[None,:,:,None]#[None,None] \n",
    "        #print(mask.shape, mask)\n",
    "        x = self.transformer(x, mask).transpose(1,2).contiguous()\n",
    "        \n",
    "        t_one_hot = torch.zeros(bs,n_types+1,n_pts,device=type.device,dtype=x.dtype).scatter_(1,type+1, 1.)\n",
    "        scalar    = self.scalar(torch.cat([x, t_one_hot], dim=1))\n",
    "        #scalar    = self.scalar(x)\n",
    "        \n",
    "        magnetic  = self.magnetic(x) \n",
    "        dipole    = self.dipole(x.view(bs,-1))\n",
    "        potential = self.potential(x.view(bs,-1))\n",
    "                \n",
    "        return type,ext,scalar,magnetic,dipole,potential\n",
    "    \n",
    "    def reset(self): pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:56:35.210939Z",
     "start_time": "2019-07-26T06:56:35.194541Z"
    }
   },
   "outputs": [],
   "source": [
    "class SecondLevelTransformer(Module):\n",
    "    def __init__(self, net1, net2):\n",
    "        self.net1 = net1\n",
    "        self.net2 = net2\n",
    "\n",
    "        self.w1 = Parameter(torch.Tensor([0.6]), requires_grad=True)\n",
    "        #self.w2 = Parameter(torch.Tensor([0.3]))\n",
    "        #self.w1.requires_grad = True\n",
    "        #self.w2.requires_grad = True\n",
    "        \n",
    "        \n",
    "    def forward(self,xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms):      \n",
    "        type1,ext1,scalar1,magnetic1,dipole1,potential1 = self.net1(xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms)\n",
    "        type2,ext2,scalar2,magnetic2,dipole2,potential2 = self.net2(xyz,type,ext,atom,mulliken,coulomb, mask_atoms, n_atoms)\n",
    "\n",
    "        self.w1.cuda()\n",
    "        scalar = (scalar1-scalar2)*self.w1 + scalar2\n",
    "        return type1,ext2,scalar,magnetic1,dipole1,potential1\n",
    "    \n",
    "    def reset(self): pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This callback allows to insert multiple stateful (not averaged) metrics in one pass. Addditionally we could add metrics for train if we want to."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Model instantiation: where's all your TPUs/GPUs when you need a decent hyperparam sweep?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:56:55.517459Z",
     "start_time": "2019-07-26T06:56:51.043042Z"
    }
   },
   "outputs": [],
   "source": [
    "net, learner = None,None\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "net1 = AtomTransformer(n_layers=6, n_heads=16,d_model=1024,d_inner=4096)\n",
    "net1.load_state_dict(torch.load('models/loss-5.9943val-2.7766.pth')['model'])\n",
    "\n",
    "net2 = AtomTransformerOld(n_layers=6, n_heads=8,d_model=512,d_inner=2048, attn_cls=MultiHeadAttentionOld)\n",
    "net2.load_state_dict(torch.load('models/loss-4.0414val-2.7287.pth')['model'])\n",
    "\n",
    "net = SecondLevelTransformer(net1, net2)\n",
    "\n",
    "learner = Learner(data,net, loss_func=LMAEMaskedLoss(),)\n",
    "\n",
    "learner.callbacks.append(LMAEMetric(learner))\n",
    "\n",
    "#for p in learner.model.parameters():\n",
    "#    if p.dim() > 1:\n",
    "#        nn.init.xavier_uniform(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:59:24.027725Z",
     "start_time": "2019-07-26T06:59:20.749967Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SecondLevelTransformer\n",
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           1,049,600  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [16, 29, 29]         0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 4096]           4,198,400  True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 4096]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1024]           4,195,328  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 1024]           0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 1024]           2,048      True      \n",
       "______________________________________________________________________\n",
       "Conv1d               [4, 29]              4,136      True      \n",
       "______________________________________________________________________\n",
       "Conv1d               [9, 29]              9,225      False     \n",
       "______________________________________________________________________\n",
       "Linear               [3]                  89,091     False     \n",
       "______________________________________________________________________\n",
       "Linear               [1]                  29,697     False     \n",
       "______________________________________________________________________\n",
       "Embedding            [29, 506]            4,554      False     \n",
       "______________________________________________________________________\n",
       "Embedding            [29, 512]            3,072      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1536]           787,968    False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            262,656    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [8, 29, 29]          0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 2048]           1,050,624  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            1,049,088  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1536]           787,968    False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            262,656    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [8, 29, 29]          0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 2048]           1,050,624  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            1,049,088  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1536]           787,968    False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            262,656    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [8, 29, 29]          0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 2048]           1,050,624  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            1,049,088  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1536]           787,968    False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            262,656    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [8, 29, 29]          0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 2048]           1,050,624  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            1,049,088  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1536]           787,968    False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            262,656    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [8, 29, 29]          0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 2048]           1,050,624  False     \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            1,049,088  False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 1536]           787,968    True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            262,656    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [8, 29, 29]          0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      True      \n",
       "______________________________________________________________________\n",
       "Linear               [29, 2048]           1,050,624  True      \n",
       "______________________________________________________________________\n",
       "ReLU                 [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 2048]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [29, 512]            1,049,088  True      \n",
       "______________________________________________________________________\n",
       "Dropout              [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "MergeLayer           [29, 512]            0          False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [29, 512]            1,024      True      \n",
       "______________________________________________________________________\n",
       "Conv1d               [4, 29]              2,052      True      \n",
       "______________________________________________________________________\n",
       "Conv1d               [9, 29]              4,617      False     \n",
       "______________________________________________________________________\n",
       "Linear               [3]                  44,547     False     \n",
       "______________________________________________________________________\n",
       "Linear               [1]                  14,849     False     \n",
       "______________________________________________________________________\n",
       "Embedding            [29, 250]            2,250      False     \n",
       "______________________________________________________________________\n",
       "Embedding            [29, 256]            1,536      False     \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 100,998,874\n",
       "Total trainable params: 16,804,396\n",
       "Total non-trainable params: 84,194,478\n",
       "Optimized with 'torch.optim.adam.Adam', betas=(0.9, 0.99)\n",
       "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
       "Loss function : LMAEMaskedLoss\n",
       "======================================================================\n",
       "Callbacks functions applied \n",
       "    LMAEMetric\n",
       "    ParallelTrainer"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T16:28:33.234417Z",
     "start_time": "2019-07-25T16:28:32.361119Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 5537584\r\n",
      "-rw-r--r-- 1 root root  50566405 Jul  8 07:27 loss-3.9978val-1.9089.pth\r\n",
      "-rw-r--r-- 1 root root  50985264 Jul  8 07:27 loss-3.9395val-1.9417.pth\r\n",
      "-rw-r--r-- 1 root root  50495731 Jul  8 07:27 loss-2.7748val-1.9216.pth\r\n",
      "-rw-r--r-- 1 root root  50500378 Jul  8 07:27 loss-4.4237val-1.9855.pth\r\n",
      "-rw-r--r-- 1 root root  50500214 Jul  8 07:27 loss-4.1809val-1.9292.pth\r\n",
      "-rw-r--r-- 1 root root  50566405 Jul  8 07:27 loss-3.8007val-1.8406.pth\r\n",
      "-rw-r--r-- 1 root root  42477068 Jul  8 07:27 loss-2.7250val-1.8349.pth\r\n",
      "-rw-r--r-- 1 root root  42477217 Jul  8 07:27 loss-2.8093val-1.8602.pth\r\n",
      "-rw-r--r-- 1 root root  50567976 Jul  8 07:27 loss-3.7100val-1.9721.pth\r\n",
      "-rw-r--r-- 1 root root  42421826 Jul  8 07:27 val-1.7450.pth\r\n",
      "-rw-r--r-- 1 root root  50987950 Jul  8 16:42 loss-3.7110val-1.9794.pth\r\n",
      "-rw-r--r-- 1 root root  51369339 Jul  9 07:00 loss-3.6805val-1.9788.pth\r\n",
      "-rw-r--r-- 1 root root  54983710 Jul 10 09:43 loss-3.8031val-1.9859.pth\r\n",
      "-rw-r--r-- 1 root root  54983942 Jul 10 09:43 loss-3.8269val-1.9933.pth\r\n",
      "-rw-r--r-- 1 root root  54986068 Jul 10 17:54 loss-3.7727val-1.9890.pth\r\n",
      "-rw-r--r-- 1 root root  54985826 Jul 12 13:07 loss-3.6655val-1.9898.pth\r\n",
      "-rw-r--r-- 1 root root  80199686 Jul 13 10:20 loss-3.9585val-2.0609.pth\r\n",
      "-rw-r--r-- 1 root root  80199266 Jul 13 15:37 loss0.1779val-2.0310.pth\r\n",
      "-rw------- 1 root root       137 Jul 13 15:40 tmpzs94ikyy\r\n",
      "-rw------- 1 root root       137 Jul 13 15:41 tmpm4w4vftj\r\n",
      "-rw-r--r-- 1 root root  80199266 Jul 13 19:46 loss0.2609val-2.0590.pth\r\n",
      "-rw-r--r-- 1 root root  80199266 Jul 14 06:32 loss0.5334val-1.1000.pth\r\n",
      "-rw-r--r-- 1 root root  80199266 Jul 14 08:41 loss0.1849val-2.5600.pth\r\n",
      "-rw-r--r-- 1 root root 227855040 Jul 15 00:53 loss-3.4306val-2.5506.pth\r\n",
      "-rw-r--r-- 1 root root  42951842 Jul 15 01:03 loss-3.3996val-2.4482.pth\r\n",
      "-rw-r--r-- 1 root root 227855204 Jul 16 02:01 loss-4.0414val-2.7287.pth\r\n",
      "-rw-r--r-- 1 root root  42951744 Jul 16 06:15 loss-3.1727val-2.4738.pth\r\n",
      "-rw-r--r-- 1 root root  42951678 Jul 16 06:15 loss-3.4640val-2.4699.pth\r\n",
      "-rw-r--r-- 1 root root 227855040 Jul 16 06:16 loss-4.0708val-2.6086.pth\r\n",
      "-rw-r--r-- 1 root root 908650352 Jul 18 06:22 loss-3.6971val-2.7181.pth\r\n",
      "-rw-r--r-- 1 root root  42780159 Jul 18 11:12 loss-3.6415val-2.5139.pth\r\n",
      "-rw-r--r-- 1 root root 984241202 Jul 23 06:11 loss-5.9943val-2.7766.pth_bak\r\n",
      "-rw-r--r-- 1 root root 984241202 Jul 23 06:12 loss-5.9943val-2.7766.pth\r\n",
      "-rw-r--r-- 1 root root 439696940 Jul 24 17:47 loss-3.1921val-2.3231.pth\r\n",
      "-rw-r--r-- 1 root root  73380929 Jul 25 15:29 tmp.pth\r\n",
      "-rw-r--r-- 1 root root 220124230 Jul 25 15:49 bestmodel.pth\r\n",
      "drwxr-xr-x 2 root root      4096 Jul 25 16:27 .\r\n",
      "drwxr-xr-x 7 root root      8192 Jul 25 16:27 ..\r\n"
     ]
    }
   ],
   "source": [
    "!ls -altr models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T16:28:55.162587Z",
     "start_time": "2019-07-25T16:28:55.158088Z"
    }
   },
   "outputs": [],
   "source": [
    "sub_fname = \"loss-5.9943val-2.7766\" # uncomment or set None to skip loading trained net\n",
    "sub_fname = \"loss-4.0414val-2.7287\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T16:29:05.472598Z",
     "start_time": "2019-07-25T16:28:55.732674Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attempting to load: loss-4.0414val-2.7287... Loaded\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    print(f\"Attempting to load: {sub_fname}... \", end=\"\")\n",
    "    learner.load(sub_fname, strict=True,with_opt=False)\n",
    "    print(\"Loaded\")\n",
    "except Exception as e:\n",
    "    print(\"NOT loaded! \", e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:56:58.928491Z",
     "start_time": "2019-07-26T06:56:58.921333Z"
    }
   },
   "outputs": [],
   "source": [
    "learner = learner.to_parallel()#.to_fp16() b/c it NaNs loss (probably would need to change fp16 settings)\n",
    "data.batch_size = int(4096//2)\n",
    "data.batch_size = data.batch_size // 2\n",
    "#data.batch_size = data.batch_size // 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Real loss func. Need to test different auxiliary tasks weights: `magnetic_w`, `dipole_w`, `potential`, weights of indivial `lmae`s: `types_w` and maybe `input_transform_w` and `feature_transform_w`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:57:00.156018Z",
     "start_time": "2019-07-26T06:57:00.149166Z"
    }
   },
   "outputs": [],
   "source": [
    "learner.loss_func = LMAEMaskedLoss(magnetic_w=0, dipole_w=0, potential_w=0, \n",
    "                                  types_w = [1] + [1] * (n_types-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T05:07:58.783101Z",
     "start_time": "2019-07-26T05:07:39.639404Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Failed to compute the gradients, there might not be enough points.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAUwUlEQVR4nO3dfbAldX3n8feHGUB5NnBNGYY4oOMmo0kAr0TDJsFFrcGtQAxGZzZugrpSbiRuSosttjZhXRIT0ayWruwqmzWiFUUeaq3RsKIhsPgAhosgwhDMSDQMsssNQRJQwoPf/aN74HDn3Dt3HvrcO/N7v6q6pvvXv+7+zrl97ud097ndqSokSe3aZ6kLkCQtLYNAkhpnEEhS4wwCSWqcQSBJjVu51AXsqCOOOKJWr1691GVI0h7lxhtv/Luqmho3b48LgtWrVzMzM7PUZUjSHiXJd+ab56khSWqcQSBJjTMIJKlxBoEkNc4gkKTGDRYEST6S5N4kt84zP0k+kGRzkluSHD9ULZKk+Q15RPBRYN0C808B1vTDmcB/H7AWSdI8BguCqroW+PsFupwGfKw61wOHJXnWUPVIksZbymsERwJ3jUxv6du2keTMJDNJZmZnZydSnCS1Yo+4WFxVF1bVdFVNT02N/QtpSdJOWsoguBs4amR6Vd8mSZqgpQyCjcCv998eejHwQFXds4T1SFKTBrvpXJJPAicBRyTZAvwnYF+AqvoQcAXwSmAz8H3g9UPVIkma32BBUFUbtjO/gLcMtX1J0uLsEReLJUnDMQgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wYNgiTrktyRZHOSc8bM//EkVye5KcktSV45ZD2SpG0NFgRJVgAXAKcAa4ENSdbO6fY7wCVVdRywHvhvQ9UjSRpvyCOCE4DNVXVnVT0CXAycNqdPAYf044cC3x2wHknSGEMGwZHAXSPTW/q2Ue8AXpdkC3AF8FvjVpTkzCQzSWZmZ2eHqFWSmrXUF4s3AB+tqlXAK4GPJ9mmpqq6sKqmq2p6ampq4kVK0t5syCC4GzhqZHpV3zbqjcAlAFV1HfA04IgBa5IkzTFkENwArElydJL96C4Gb5zT52+BkwGS/CRdEHjuR5ImaLAgqKrHgLOAK4Hb6b4ddFuS85Kc2nd7O/CmJF8HPgmcUVU1VE2SpG2tHHLlVXUF3UXg0bZzR8Y3AScOWYMkaWFLfbFYkrTEDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYNGgRJ1iW5I8nmJOfM0+c1STYluS3JJ4asR5K0rZVDrTjJCuAC4OXAFuCGJBuratNInzXAfwBOrKr7kzxzqHokSeMNeURwArC5qu6sqkeAi4HT5vR5E3BBVd0PUFX3DliPJGmMIYPgSOCukektfduo5wHPS/LlJNcnWTduRUnOTDKTZGZ2dnagciWpTUt9sXglsAY4CdgA/I8kh83tVFUXVtV0VU1PTU1NuERJ2rsNGQR3A0eNTK/q20ZtATZW1aNV9TfAN+mCQZI0IUMGwQ3AmiRHJ9kPWA9snNPn03RHAyQ5gu5U0Z0D1iRJmmOwIKiqx4CzgCuB24FLquq2JOclObXvdiVwX5JNwNXA2VV131A1SZK2lapa6hp2yPT0dM3MzCx1GZK0R0lyY1VNj5u31BeLJUlLzCCQpMYZBJLUOINAkhpnEEhS4wwCSWrcooIgyXOS7N+Pn5TkreNuBSFJ2vMs9ojgcuDxJM8FLqS7dYTPDpCkvcBig+CH/V8Kvwr4r1V1NvCs4cqSJE3KYoPg0SQbgN8APtu37TtMSZKkSVpsELweeAnwzqr6myRHAx8frixJ0qQs6lGV/eMl3wqQ5BnAwVV1/pCFSZImY7HfGromySFJfgT4Gt0DZN47bGmSpElY7KmhQ6vqH4BfAT5WVT8LvGy4siRJk7LYIFiZ5FnAa3jyYrEkaS+w2CA4j+4hMt+qqhuSHAP89XBlSZImZbEXiy8FLh2ZvhM4faiiJEmTs9iLxauS/K8k9/bD5UlWDV2cJGl4iz019Cd0D57/sX74TN8mSdrDLTYIpqrqT6rqsX74KDA1YF2SpAlZbBDcl+R1SVb0w+uA+4YsTJI0GYsNgjfQfXX0/wL3AK8GzhioJknSBC0qCKrqO1V1alVNVdUzq+qX8VtDkrRX2JUnlL1tt1UhSVoyuxIE2W1VSJKWzK4EQe22KiRJS2bBvyxO8o+M/4Uf4OmDVCRJmqgFg6CqDp5UIZKkpbErp4YkSXsBg0CSGmcQSFLjBg2CJOuS3JFkc5JzFuh3epJKMj1kPZKkbQ0WBElWABcApwBrgQ1J1o7pdzDw74CvDlWLJGl+Qx4RnABsrqo7q+oR4GLgtDH9fg84H3h4wFokSfMYMgiOBO4amd7Stz0hyfHAUVX1ZwutKMmZSWaSzMzOzu7+SiWpYUt2sTjJPsB7gbdvr29VXVhV01U1PTXlYxAkaXcaMgjuBo4amV7Vt211MPAC4Jok3wZeDGz0grEkTdaQQXADsCbJ0Un2A9bTPe4SgKp6oKqOqKrVVbUauB44tapmBqxJkjTHYEFQVY8BZwFXArcDl1TVbUnOS3LqUNuVJO2YBe81tKuq6grgijlt587T96Qha5EkjedfFktS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGDRoESdYluSPJ5iTnjJn/tiSbktyS5Kokzx6yHknStgYLgiQrgAuAU4C1wIYka+d0uwmYrqqfBi4D3j1UPZKk8YY8IjgB2FxVd1bVI8DFwGmjHarq6qr6fj95PbBqwHokSWMMGQRHAneNTG/p2+bzRuB/j5uR5MwkM0lmZmdnd2OJkqRlcbE4yeuAaeA94+ZX1YVVNV1V01NTU5MtTpL2cisHXPfdwFEj06v6tqdI8jLgPwK/WFX/NGA9kqQxhjwiuAFYk+ToJPsB64GNox2SHAd8GDi1qu4dsBZJ0jwGC4Kqegw4C7gSuB24pKpuS3JeklP7bu8BDgIuTXJzko3zrE6SNJAhTw1RVVcAV8xpO3dk/GVDbl+StH3L4mKxJGnpGASS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcYMGQZJ1Se5IsjnJOWPm75/kU/38ryZZPWQ9kqRtDRYESVYAFwCnAGuBDUnWzun2RuD+qnou8D7g/KHqkSSNN+QRwQnA5qq6s6oeAS4GTpvT5zTgon78MuDkJBmwJknSHEMGwZHAXSPTW/q2sX2q6jHgAeDwuStKcmaSmSQzs7OzA5UrSW3aIy4WV9WFVTVdVdNTU1NLXY4k7VWGDIK7gaNGplf1bWP7JFkJHArcN2BNkqQ5hgyCG4A1SY5Osh+wHtg4p89G4Df68VcDf1FVNWBNkqQ5Vg614qp6LMlZwJXACuAjVXVbkvOAmaraCPxP4ONJNgN/TxcWkqQJGiwIAKrqCuCKOW3njow/DPzqkDVIkha2R1wsliQNxyCQpMYZBJLUOINAkhqXPe3bmklmge8sdR1zHAH83VIXMcdyrAmWZ13LsSZYnnUtx5pgeda13Gp6dlWN/YvcPS4IlqMkM1U1vdR1jFqONcHyrGs51gTLs67lWBMsz7qWY03z8dSQJDXOIJCkxhkEu8eFS13AGMuxJliedS3HmmB51rUca4LlWddyrGksrxFIUuM8IpCkxhkEktS45oMgyUeS3Jvk1h1c7oAkf5bkr5LcluRdI/Pel+Tmfvhmku+NzDs/ya398NpJ1jXS5/QklWS6n355khuTfKP/919MqqYk+yf5VJLNSb6aZHXfvjrJD0Zexw8tsP6dqqtf9p1J7kry4Jh5r0myqa/5EyPt7+7bbk/ygXGPVx2ipiRv7n9GNyf50tZngCc5PMnVSR5M8sEd2M6u1PjaJLf0r8M2zxqfu49NqKZrktwxss88s2+f9/04dF07s+8viapqegB+ATgeuHUHlzsAeGk/vh/wReCUMf1+i+4W3AD/EvgC3V1fD6R7ZsMhk6wLOBi4FrgemO7bjgN+rB9/AXD3pGoCfhP4UD++HvhUP756sdvZ2br6ZV8MPAt4cE77GuAm4Bn99DP7f38O+DLdrdVXANcBJ02opkNGxk8FPtePHwj8c+DNwAcnsO8fDvwtMNVPXwScvNA+NnRN/bLXbG97o+/HSdS1M/v+UgzNHxFU1bV0z0J4QpLnJPlc/+n4i0l+Ysxy36+qq/vxR4Cv0T2Fba4NwCf78bXAtVX1WFU9BNwCrJtwXb8HnA88PLLMTVX13X7yNuDpSfafUE2n0f0iAbgMOHncJ+yF7Gxd/bLXV9U9Y2a9Cbigqu7v+927dRHgaXRv6v2BfYH/N4maquofRiYP7Guhqh6qqi8x8jNdjF2o8Rjgr6tq6wPE/xw4fWT+NvvYBGparNH34+B1Db3v7zZLlUDLaWDOp0/gKmBNP/6zdE9OW2j5w4A7gWPmtD8buAdY0U+/gu7T5AF0f35+J/D2SdVF94nm8n78GsZ8eqJ7UtyfT7CmW4FVI/O/1b82q4GH6D6V/x/g5wf+Gc799P1p4N39z+t6YN3IvD8Cvgc8ALxzUjX1bW/pX6O7tq5rZN4Z7MARwc7WCDwD2NIvuxK4HPjMYvexIWoa2d43gJuB36X/VuTI/Ke8HydV147u+ztT264Ogz6YZk+U5CC6w/9LR8J5m0/HI/1X0n3C+EBV3Tln9nrgsqp6HKCqPp/kRcBXgFm60wqPT6KuJPsA76X7ZTHfMs+n+yT3iknUtJ3V3wP8eFXdl+SFwKeTPL+e+ql4t9Q1j5V0p4dOovsEd22Sn6ILqZ/kyU91X0jy81X1xQnURFVdAFyQ5F8Bv8OTj3rdZYutsaruT/JvgU8BP6Tbn5+zmH1sqJp6v1ZVdyc5mC6c/jXwsZH5T3k/TrCuHd33J28p0me5DYwkPXAIcM+YPivoPmncDJw30v4Ruh/uuPXeBPzcAtv9BPDKSdQFHEp3A6xv98PDwHd58jrBKuCbwImTfK3oHmX6kn58ZV9jxqzzGhb4dLkrdfXz5h4RfAh4/cj0VcCLgLOB3x1pPxf495Ooac68fYAH5rSdwS4cEexMjf38M+mOnhbcxyZc0zavBdt5Pw5Z167s+5MYJr7B5Tiw7SHfV4Bf7ccD/Mw8y/0+3SePfcbM+4n+zZCRthXA4f34T9MdGq6cZF0jfa7hyRA4DPg68CuTfq3oTnWMXjC7pB+f4slTascAdwM/srvrGuk/NwjWARf140fQnYo5HHgt3TnxlXTXB64CfmlCNa0ZGf8lumd/j84/g10/NbTYn+fWi+fPoPul97yF9rGha+p/Hkf04/vSnXN/88j8bd6Pk6hrZ/b9pRiWZKPLaaA7XLsHeJTuvOcbgaOBz9H9ctwEnDtmuVV0F+tu58lPAP9mZP47gHfNWeZp/fo20Z13PnbSdY30e+JNSneK4aGR/jdvfaMPXVP/mlwKbAb+kifPn55Od+H6ZroLbGN/2e5KXf2y7+6X+WH/7zv69tCd5thEd955fd++Avhw/3/ZBLx3gjW9f+Q1uRp4/sgy36a7mPlgv8zaofb9kWW37svr5+nzxD42gffjgcCNdF/AuK1/rVaMzH8Hc96PS/17gnn2/aUYvMWEJDWu+a+PSlLrDAJJapxBIEmNMwgkqXEGgSQ1ziDQHm/uXTonsL0/3nrnz92wrsf7u2LemuQzSQ7bTv/Dkvzm7ti2tJVfH9UeL8mDVXXQblzfyqp6bHetbzvbeqL2JBcB36yqdy7QfzXw2ap6wSTqUxs8ItBeKclUksuT3NAPJ/btJyS5LslNSb6S5J/17Wck2ZjkL4CrkpzU39/+sv5e8n+69c6QffvWZzk8mO75AV9Pcn2SH+3bn9NPfyPJ7y/yqOU64Mh++YOSXJXka/06Tuv7vIvuvj43J3lP3/fs/v94S5L/vBtfRjXCINDe6v3A+6rqRXR/pfzHfftf0d3J9Di6+wT9wcgyxwOvrqpf7KePA36b7vbhxwAnjtnOgcD1VfUzdPfgf9PI9t9fVT9F95eoC0qyAjgZ2Ng3PQy8qqqOB14K/Jc+iM4BvlVVx1bV2UleQXdzvBOAY4EXJvmF7W1PGuXdR7W3ehmwduTOkIf0d4w8FLgoyRq6P/3fd2SZL1TV6D3n/7KqtgAkuZnuXjNfmrOdR4DP9uM3Ai/vx18C/HI//gm6W1eP8/R+3UfS3YbgC317gD/of6n/sJ//o2OWf0U/3NRPH0QXDNfOsz1pGwaB9lb7AC+uqqc8HCXdoxyvrqpX9efbrxmZ/dCcdfzTyPjjjH+/PFpPXmibr89CflBVxyY5gO5ulG8BPgD8Gt2N915YVY8m+TbdvWnmCvCHVfXhHdyu9ARPDWlv9Xm6xxICkOTYfvRQujuZwm68b/4Y1/PkU7vWb69zVX0feCvw9v7e9YcC9/Yh8FK6h6oA/CPdoyC3uhJ4Q3+0Q5Ij0z+rV1osg0B7gwOSbBkZ3kb3S3W6v4C6ie55vtDd2fMPk9zEsEfEvw28LcktwHPpnma2oKq6ie7umRuAP6Wr/xvAr9Nd26Cq7gO+3H/d9D1V9Xm6U0/X9X0v46lBIW2XXx+VBtCf6vlBVVWS9cCGqjpte8tJS8FrBNIwXgh8sP+mz/eANyxxPdK8PCKQpMZ5jUCSGmcQSFLjDAJJapxBIEmNMwgkqXH/H+4qMsuC5AQ7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.lr_find()\n",
    "learner.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:44:49.070950Z",
     "start_time": "2019-07-26T06:44:31.754136Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='100', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/100 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>👉🏻LMAE👈🏻</th>\n",
       "      <th>lmae0</th>\n",
       "      <th>lmae1</th>\n",
       "      <th>lmae2</th>\n",
       "      <th>lmae3</th>\n",
       "      <th>lmae4</th>\n",
       "      <th>lmae5</th>\n",
       "      <th>lmae6</th>\n",
       "      <th>lmae7</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='136', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-77-3cd29c879174>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#, moms=(0.75,0.70))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mto_detach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-a8bc1c69b8fd>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_outputs, t_scalar, t_magnetic, t_dipole, t_potential)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_types\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexclude_ext\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mext\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m                 \u001b[0m_output\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_target\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp_scalar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt_scalar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# scalars at the end\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                 \u001b[0;31m# LMAE scalar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learner.fit(100, 1e-2)#, moms=(0.75,0.70))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T06:19:21.181522Z",
     "start_time": "2019-07-18T06:19:20.588341Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuMAAAEGCAYAAADPHJsIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXhU9fXH8ffJvhASSEKAJOxhCQQQIoqAoKggKih1X6rV1i5arVvdtWpd2rrVVq222rq0KqIoKooLyKKAgrIkQEIIWwghIUASAlnn/P7I4C+lIRkgyZ3lvJ7nPkzu3HvzmQAzJ/d+7/mKqmKMMcYYY4xpf0FOBzDGGGOMMSZQWTFujDHGGGOMQ6wYN8YYY4wxxiFWjBtjjDHGGOMQK8aNMcYYY4xxSIjTAZyUkJCgvXr1cjqGMcYcsRUrVuxS1USnc7Qne882xviq5t6zA7oY79WrF8uXL3c6hjHGHDER2eJ0hvZm79nGGF/V3Hu2DVMxxhhjjDHGIVaMG2OMMcYY4xArxo0xxhhjjHGIFePGGGOMMcY4xIpxY4wxxhhjHNKmxbiITBaRHBHJE5E7mng+XETecj+/TER6NXruTvf6HBGZ1Gj9yyJSLCJZhxyrs4h8JiIb3H92asvXZowxxhhjzLFqs2JcRIKBZ4EzgXTgEhFJP2Sza4A9qtoPeAr4g3vfdOBiYDAwGXjOfTyAf7nXHeoO4AtVTQO+cH9tjDHGGGOM12rLPuOjgDxVzQcQkTeBacDaRttMA37nfjwT+KuIiHv9m6paDWwSkTz38Zao6sLGZ9APOdYE9+NXgC+B21vv5ZjW4HIpa3eUs3LbXvZU1gAQFR5CVFgw8dFhJHeKJKVTFLGRoQ4nNcb4g5cWb6Ku3kWPzlGM7htPXFSY05GMMT6guq6egj0H2Lp7P6X7aqioqmVfVR019S7OGtqNgV07ttr3astiPBnY1ujrAuCEw22jqnUiUgbEu9cvPWTf5Ba+X5Kq7nA/LgKSmtpIRK4FrgXo0aNHy6/CtIq6ehdvfLuNlxbls7l0f4vbd4kJJyM5liHJsZzQuzMje3UiPCS4xf2MMaax15duYdOuSgBCgoSzhnbjxolp9Ens4HAyY4y3cLmUrMIyvt28h1Xb9rK6YC9bdu9H9X+3FYH+STE+U4w7RlVVRJr4EYKqvgi8CJCZmdnkNqZ1bdhZwQ1vrmTdjnJG9IjjulP6MbpvPEkdIwDYX1NPZXUdJRXVbN97gG2797O+qIKs7WXMzynmzwoRoUGc2CeeM9K7MiWjq53dMsZ4ZN4t46msqSenqJyPVhcxY/k2Pl5TxE2n9+cX4/vQcDHWGBNoquvqmb++mE/X7mRhbgm79jVcre8WG8GwlDimDU+mZ3wUqZ2j6BITTkxEKB3CQwgLaf0R3m1ZjG8HUht9neJe19Q2BSISAsQCpR7ue6idItJNVXeISDeg+FjCm9bx2dqd/ObN74kMC+b5y0YweUjX//nwi40MIjYylO5xkQxLjfuv5/ZV17Esv5RFG3axILeEu2at4f7ZWYzvn8iFmalMHJREcJB9mBpjmiYidAgPYWTPzozs2ZlfjO/Dfe9n84dP1pNVWMYTFwwjItSuuhkTKFZu28vby7fx4eodlB2oJS4qlPH9E5kwIJGT+ib8cKKwPbVlMf4tkCYivWkopC8GLj1km9nAlcAS4Hxgnvus9mzgPyLyJNAdSAO+aeH7HTzWY+4/32+tF2KOzkerd3DDm98zpHtHXrgik66xR/4PvEN4CBMHJTFxUBKqSnZhObNXFfL+yu18vq6YlE6RXHFiTy4+vgexUTbO3BjTvC4dI3j+8hG8sDCfxz5ez4Gaep6/fIQNgzPGj9XVu5ibvZOXFufz3da9RIQGMXlwV6aPSOGkvvGEBDvb6Vu0qQExrXVwkSnA00Aw8LKqPiwiDwLLVXW2iEQArwHHAbuBixvd8Hk3cDVQB/xGVT92r3+Dhhs1E4CdwP2q+pKIxAMzgB7AFuBCVd3dXL7MzExdvnx5a79sA8xfX8xPX13OiB5x/PMno+gQ3rq/99XVu/hs7U7+9fVmlm3aTUx4CFeP7c3VY3vbzZ8mIIjIClXNdDpHe2rt9+z/LNvKXbPWcFZGN/566XE2ZMUYP1PvUmZ9v50/f5HLtt0H6BkfxdVjevOjkSmtXpe0pLn37DYtxr2dFeNtI3dnBdOf+5qe8VHM+Plootv4H3x2YRnPfLGBudk76RgRws/H9+Wasb3t0rPxa1aMt44XF27kkTnrueX0/vx6YlqrHtsY4wxV5ZOsIp74LJe84n1kJMdy3Sn9OD3duaGtzb1n++UNnMY5FVW1/OzV5USGBfOPKzPbvBAHGNw9lheuyCRrexlPf57Ln+bm8Oa3W7l7SjqTBifZ2S5jzGH9bFwf1u+o4InPchmaGsf4/olORzLGHIOcogrufT+Lbzbtpl+XDvzt8hFMGvy/96t5E2cHyRi/88AHa9m2ez/PXTaCbrGR7fq9hyTH8o8rj+ffPz2ByNBgfvH6Cn788jds291yK0VjTGASER6ZnkFalw7c9vaqH+Y/MMb4ln3Vdfz+w7VMeWYRuTsreOS8DOb+5mQmD+nm1YU4WDFuWtHHa3Ywc0UB153Sj+N7dXYsx5h+Ccy5YRz3n5PO91v3Munphby6ZDMuV+AOyTLGHF5EaDBPXTSc3ZU13PN+ltNxjDFHaMnGUiY9tZCXvtrEhZkpzLtlApee0MNnuq1ZMW5aRdmBWu55L4uM5Fhu8IJxlyHBQfxkTG/m3nQyI3t24r73s7nk70vtLLkxpklDkmO5cWIaH63ewfwc64xrjC+oqq3ngQ8aPt9Dg4WZvxjNo9OH0jnat+YisWLctIonPs1hz/4aHp2eQajDLYIaS46L5NWrR/GHH2WQXVjOWc8s4pOsIqdjGWO80M/H96VPYjS/m51NVW2903GMMc3IKarg7L8s5p9fbebK0T2Zc+M4RvZ07qr8sfCeqsn4rKztZby+dAtXnNiTIcmxTsf5HyLCRcf34KMbxtIzPppfvL6C+9/Psg9bY8x/CQsJ4oGpg9lSup+/L8x3Oo4x5jDeWVHAtGcXU3agltevOYEHpg0hKsx3e5JYMW6OiarywAfZdI4O4+YzBjgdp1k946N555cncc3Y3ryyZAsXvrCEHWUHnI5ljPEi49ISmTy4K39bsJHSfdVOxzHGNFJVW88d76zmlrdXMTw1jo9uGMvYtASnYx0zK8bNMZmfU8y3m/dw0+n9fWKynbCQIO49O50XrxjJxuJ9TP3rV3y3dY/TsYwxXuTWSQM4UFvPs/M3Oh3FGONWXF7FRS8s4c1vt3HdKX15/ZoT6BLT/lPXtwUrxs1Rc7mUP36SQ6/4KC7MTHU6zhE5Y3BXZl03hsjQYC5+YSkzVxQ4HckY4yX6denA+SNTeH3pFrbvtatnxjhtbWE55z77Fbk79/HCFSO5bdJAx6ewb03+80pMu/tgdSHriyq46fT+XnXTpqf6J8Xw/nVjGNmzE7e+vYonP80hkGekNcb8vxtP6w/AX77Y4HASYwLb52t3cv7fvsal8PYvRjNpcFenI7U636ugjFeodylPf76BQd06cs7Q7k7HOWqdosN49ZpRXJiZwjPz8rjjnTXU1bucjmWMcVhyXCQXHZ/KO98VUFRW5XQcYwLSv5dt4WevLadvYgfev36MVzaJaA1WjJuj8klWEZt2VfLrU/sR5CNN9Q8nNDiIP/xoKL8+tR9vLd/Gz19bwYEa67RijCdEZLKI5IhInojc0cTzPUXkCxFZLSJfikjKIc93FJECEflr+6X2zLUn98Gl8PJXm5yOYkxAUVWenZ/H3bOymNA/kbd+fiJJHf1jfHhTrBg3R0xV+duCjfROiPaby0Uiwi1nDOChaYOZl1PMZf9YStmBWqdjGePVRCQYeBY4E0gHLhGR9EM2exx4VVWHAg8Cjx7y/EPAwrbOejRSO0dx9tBu/HvpFsr22/uBMe1BVXlkzjr+NDeHacO78+KPM326baEnrBg3R+yrvFLWbC/j2pP7+MxUs566YnQvnrt0BGu2l3HFS8vYu7/G6UjGeLNRQJ6q5qtqDfAmMO2QbdKBee7H8xs/LyIjgSTg03bIelR+Mb4vlTX1vLpks9NRjPF79S7l9ndW8/dFm7hydE+eunC4T96TdqT8/xWaVvf8gjy6xIQzfUSy01HaxJkZ3fjb5SNZv6OCS/++jN2VVpAbcxjJwLZGXxe41zW2CpjufnweECMi8SISBDwB3NrcNxCRa0VkuYgsLykpaaXYnhvUrSPj+yfy6tIt1NTZ/STGtJV6l3LbzFXMWF7ADaf243dTB/v8MFhPWTFujsjawnK+yivlJ2N6Ex4S7HScNjNxUBJ/vzKTjSX7uOTFpeyyyT+MOVq3AuNF5HtgPLAdqAd+BcxR1Wb7iqrqi6qaqaqZiYmJbZ+2CVeN6UVJRTVzs4sc+f7G+DuX+4z4u99t5+bT+3PzGQMQCYxCHKwYN0fotaWbCQ8J4pJRvtVX/GiM75/Iy1cdz5bdlVz696V2htyY/7UdaPxmkOJe9wNVLVTV6ap6HHC3e91eYDRwvYhspmFc+Y9F5LF2SX2Exqcl0jM+yoaqGNMGDhbiM1cUcNNp/blhYprTkdqdFePGY2UHannv+0KmDe9OXFSY03HaxZh+Cbx81fFsLt3PlS9/Q0WV3cRlTCPfAmki0ltEwoCLgdmNNxCRBPeQFIA7gZcBVPUyVe2hqr1oOHv+qqr+TzcWbxAUJFxxYk++3byHtYXlTscxxm+4XMpds9bw9ooCbpyYxo2nBV4hDlaMmyMwc0UBB2rr+fHoXk5HaVcn9U3gb5ePYN2Ocq7513Jre2iMm6rWAdcDc4F1wAxVzRaRB0VkqnuzCUCOiOTScLPmw46EPUYXjEwlIjSI15ZudjqKMX7hYNeUN7/dxg2n9uOm0/s7HckxVowbj7hcyutLtzCiR5zfNt1vzqkDk3j64uEs37Kbn7++guo6K8iNAVDVOaraX1X7qurD7nX3qeps9+OZqprm3uanqvo/N2Co6r9U9fr2zn4kYqNCOXd4MrO+3065XSEz5pg9v2Aj/1i8iatO6hXQhThYMW48tDhvF5t2VQbcWfHGzh7ancemD2Vhbgk3vbWSepc6HckY044uHtWDqloXH63e4XQUY3zaG99s5Y+f5HDu8O7cd3Z6QN2s2RQrxo1H3vhmK/HRYZyZ4R+T/BytC49P5Z6zBjFnTREPf7TO6TjGmHY0LCWW/kkdmLF8W8sbG2Oa9EnWDu6etYYJAxL50wXDAqZ9YXOsGDct2l1Zw+frdnLuccl+3c7QUz8d14erx/Tm5a828Y9F+U7HMca0ExHhgpGpfL91L3nFFU7HMcbnLMsv5YY3VjI8NY7nLhsREBP6eMJ+CqZF76/cTm29ckFmitNRvMY9Zw1iSkZXfv/ROj5cXeh0HGNMOzn3uGRCgoS3lzfbHt0Yc4j8kn1c+9oKUjtH8vJVx/v9FPdHwopx06IZywvISI5lYNeOTkfxGkFBwpMXDuf4Xp24+a1VLMsvdTqSMaYdJMaEc+rALrzz3XZq621GTmM8sbuyhqv/9S0hQcI/rxoVMO2RPWXFuGlW1vYy1u0ot7PiTYgIDebvP84kpXMk1762gk27Kp2OZIxpBxdkprJrXzVf5pQ4HcUYr1ddV8/PX1tOYVkVL/44kx7xUU5H8jpWjJtmzVxRQFhwEFOHdXc6ileKiwrjX1eNIkjgmle+peyAtTwzxt9NGJBI5+gw3l+5veWNjQlgqsrtM1fz7eY9PHHBMEb27OR0JK9kxbg5rOq6et5buZ3TByfZJaVm9IiP4vnLR7K1dD83vPG9tTw0xs+FBgcxJaMrn6/bSWV1ndNxjPFaz3yRx3srC7lt0gDOsZN6h2XFuDmsL3NK2Lu/lvNH2hCVlpzYJ54Hpw1hQW4Jj86xlofG+Lupw5KpqnXx2dqdTkcxxivNzS7iqc9zmT4imV9N6Ot0HK9mxbg5rNmrComPDmNcvwSno/iES0/owZWje/KPxZusD7Exfi6zZye6x0Ywe5V1UzLmUHnFFdz81kqGpcTyyHkZAT+pT0usGDdNqqyu44t1O5mS0Y0Q6wPqsXvPTmdMv3junrWGFVt2Ox3HGNNGgoKEc4Z1Z2FuCXsqa5yOY4zXKDtQy89eXUFkWDB/u2IkEaE2P0lL2rTKEpHJIpIjInkickcTz4eLyFvu55eJSK9Gz93pXp8jIpNaOqaITBSR70RkpYgsFpF+bfna/N1na3dSVeti6nAb43UkQoKDePbSEXSPi+RX//6OkopqpyMZY9rIOcO6U+dS5mTtcDqKMV7B5VJuemsl23bv57nLRtItNtLpSD6hzYpxEQkGngXOBNKBS0Qk/ZDNrgH2qGo/4CngD+5904GLgcHAZOA5EQlu4ZjPA5ep6nDgP8A9bfXaAsHsVYV0j41gZA+78/lIxUWF8fxlIyk7UMuv3/iOOutFbIxfGty9I30To5m90oaqGAPw1Oe5zFtfzP3npDOqd2en4/iMtjwzPgrIU9V8Va0B3gSmHbLNNOAV9+OZwERpGFg0DXhTVatVdROQ5z5ec8dU4OCsNLGAvTsepT2VNSzMLeGcYd0JCrJxXkcjvXtHHj43g6X5u/nT3Byn4xhj2oCIMHVYMt9s3k1xeZXTcYxx1NzsIv4yL4+LMlO5/MSeTsfxKW1ZjCcDje9iK3Cva3IbVa0DyoD4ZvZt7pg/BeaISAFwBfBYU6FE5FoRWS4iy0tKbMKGpnycVUSdS60N0TH60cgULjuhBy8szOcTu4xtjF+aktEVVZhrXVVMANtSWsmtb69iWEosD5472G7YPEL+dGfeTcAUVU0B/gk82dRGqvqiqmaqamZiYmK7BvQVs1dtp09CNIO7d2x5Y9Os+85JZ1hqHLe+vZqNJfucjmOMaWX9unSgT2K0/cJtAlZVbT3X/ec7BPjrpSMID7EbNo9UWxbj24HURl+nuNc1uY2IhNAwvKS0mX2bXC8iicAwVV3mXv8WcFLrvIzAsrO8imWbdnPOsO72m20rCA8J5vnLRhAWEsQvX1/B/hqbIMQYfyIinDmkK0vzd1tXFROQHv5oHVnby3niwuGkdrap7o9GWxbj3wJpItJbRMJouCFz9iHbzAaudD8+H5inqupef7G720pvIA34pplj7gFiRaS/+1inAzbzylGYm12EKpw9tJvTUfxG97hInrn4ODYU7+P+97OdjmOMaWWTB3ej3qV8ts6GqpjAMntVIa8t3cLPxvXm9PQkp+P4rDYrxt1jwK8H5tJQGM9Q1WwReVBEpro3ewmIF5E84GbgDve+2cAMYC3wCXCdqtYf7pju9T8D3hGRVTSMGb+trV6bP/skq4i+idGkJcU4HcWvjE1L4PpT+vH2igLe+/7QC0TGGF82JLkjKZ0i+SSryOkoxrSbjSX7uPOd1YzoEcdvJw90Oo5PC2nLg6vqHGDOIevua/S4CrjgMPs+DDzsyTHd62cBs44xckDbXVnDsk27+eV4m7a2Ldw4MY2l+aXcPWsNw1Lj6J0Q7XQkY0wrEBEmD+7Kq0u2UFFVS0xEqNORjGlTVbX1XPfv7wgLCeKvl44g1CYHPCb20zM/+HzdTupdyuQhXZ2O4pdCgoP488XHERIcxK/f+I7qunqnIxljWsnkIV2pqXcxb32x01GMaXMPfLCW9UUVPHnRcLrH2cQ+x8qKcfODuVlFJMdFWheVNtQ9LpLHLxhG1vZyHvt4vdNxjDGtZESPTiTGhDM324aqGP/2SVYRb3yzlZ+f3IdTBnRxOo5fsGLcALCvuo5FG3YxeUhX66LSxk5PT+Kqk3rxz68285n1JjbGLwQFCZMGJzF/fQlVtXbVy/inorIq7nh3NRnJsdxyxgCn4/gNK8YNAPPXF1NT77IhKu3kzikDGdy9I7fNXMWOsgNOxzHGtILT07tyoLaeJRtLnY5iTKurdyk3vbWS6loXf754OGEhVkK2FvtJGgA+yS4ioUM4I3p0cjpKQAgPCeavl46gps7FLTNW4XKp05GMMcfohN6diQoL5nNrcWj80IsL81mSX8rvpqbTJ7GD03H8ihXjhqraeuavL2bS4CSCg2yISnvpnRDNfWen8/XGUl7+apPTcYwxxygiNJhxaQnMW19Mw5QZxviH1QV7eeLTHKZkdOXCzNSWdzBHxIpxw+INu9hfU29DVBxw0fGpnDYoiT9+ksP6onKn4xhjjtHEQUnsKKsiu9D+Pxv/UFldx41vriQxJpxHzxtq95W1ASvGDXOzi+gYEcKJfeKdjhJwRIQ//CiDjpGh/ObNlXbjlzE+7tSBXRCBL9ZZi0PjHx74IJvNpZU8ddFwYqOsh35bsGI8wNW7lHnrizllYBdr2u+Q+A7h/PH8DNYXVfDEpzlOxzHGHIOEDuEMT43ji/U2btz4vrnZRcxYXsAvx/e1E3ZtyKqvALeqYC+llTWcOtB6hTrp1IFJXH5iD/6xeBNf5+1yOo4x5hicNiiJ1QVl7CyvcjqKMUdt175q7np3DYO7d+Q3p/V3Oo5fs2I8wH2xbifBQcKE/laMO+3uKen0jo/mlrdXUba/1uk4xrRIRCaLSI6I5InIHU0831NEvhCR1SLypYikuNcPF5ElIpLtfu6i9k/fdiYOang/tdk4ja9SVe6etYaKqjqevNDaGLY1++kGuC/WFXN8r042DswLRIYF8/TFwympqObe97OcjmNMs0QkGHgWOBNIBy4RkfRDNnsceFVVhwIPAo+61+8Hfqyqg4HJwNMiEtc+ydvegKQYkuMibdy48VnvrdzO3Oyd3HJGfwZ0jXE6jt+zYjyAFezZz/qiCiYOTHI6inEbmhLHDRPTmL2qkDlrdjgdx5jmjALyVDVfVWuAN4Fph2yTDsxzP55/8HlVzVXVDe7HhUAxkNguqduBiHDaoC4szrPZOI3v2VF2gPvezyazZyd+Oq6P03ECghXjAezgJdSDl1SNd/jlhL5kJMdyz3tZ7NpX7XQcYw4nGdjW6OsC97rGVgHT3Y/PA2JE5L/uAhORUUAYsLGNcjpi4qAkqmpdfL3R7gExvkNV+e3M1dTVK49fMMzmHmknVowHsM/XFdMnIdpm0vIyocFBPH7BMPZV1XHve1k2eYjxZbcC40Xke2A8sB344VSxiHQDXgN+oqqupg4gIteKyHIRWV5SUtIemVvFCX06ExkazIIc38lszL+XbWXRhl3cddYgeiVEOx0nYFgxHqD2VdexdGOpnRX3UgO6xvCb09P4OKuID1fbcBXjlbYDjafiS3Gv+4GqFqrqdFU9DrjbvW4vgIh0BD4C7lbVpYf7Jqr6oqpmqmpmYqLvjGQJDwnmpL7xfJlrxbjxDVtKK3lkzjrGpSVw+Qk9nI4TUKwYD1CLN5RQU+/iVBsv7rWuHdeHYalx3Pd+FiUVNlzFeJ1vgTQR6S0iYcDFwOzGG4hIgogc/Jy5E3jZvT4MmEXDzZ0z2zFzuxo/IJEtpfvZvKvS6SjGNKvepdwyYxXBQcIfz7dZNtubFeMB6vN1xXSMCCGzVyeno5jDCAkO4vHzh1JZU889762x4SrGq6hqHXA9MBdYB8xQ1WwReVBEpro3mwDkiEgukAQ87F5/IXAycJWIrHQvw9v3FbS9gy1jv8yxrirGu/3zq00s37KHB6YOpltspNNxAk6I0wFM+3O5lPnri5kwwGbd9HZpSTHccnp/Hv14PbNXFTJt+KH3xxnjHFWdA8w5ZN19jR7PBP7nzLeqvg683uYBHdYjPoreCdF8mVvCVWN6Ox3HmCZt2lXJn+bmcNqgJM47zj5jnGCVWABa6Z5108aL+4afjuvDcT3iuO/9bIptRj9jfMr4/okszS+1FofGK7lcyu3vrCYsJIiHzxtiw1Mc4lExLiJjReQn7seJImK/4vswm3XTtwQHCY9fMIyq2nrummXDVYzxJeMHJFJV62LZpt1ORzHmf/z7m618s2k3956VTlLHCKfjBKwWi3ERuR+4nYabbwBCCYDLi/5s3voSRva0WTd9Sd/EDtw2aQCfryvm/ZWFTscxxnhodJ94wkOCrMWh8ToFe/bzmLt7ygWZKU7HCWienBk/D5gKVMIPs6XZ3Kg+amd5Fet2lDNhgO+0CDMNfjKmN8NT43jgg2xKbTIgY3xCRGgwJ/SJ58tcu4nTeA9V5c5316DAI+dl2PAUh3lSjNdow3VxBRAR6wLvwxa6e97aEBXfc7Dl1L7qOh74YK3TcYwxHprQP5H8kkq27d7vdBRjAJi5ooBFG3Zxx5kDSe0c5XScgOdJMT5DRF4A4kTkZ8DnwD/aNpZpK1/mlpAYE86gbnZxwxf1T4rhulP6MXtVIV+s2+l0HGOMB8a7r0TaBEDGGxSXV/HQh2sZ1aszl5/Q0+k4Bg+KcVV9nIbWVO8AA4D7VPWZtg5mWl9dvYvFG3Yxvn+iXZLyYb+a0I8BSTHcPSuLiqpap+MYY1rQJyGa1M6RLLB+48Zhqsrd72VRXefiD+cPJSjIagFv4MkNnH9Q1c9U9TZVvVVVPxORP7RHONO6VhWUUXag1saL+7iwkCAe+1EGOyuqeOzj9U7HMca0QKShe9XXG0uprrMWh8Y5H67ewWdrd3LLGf3pnWCjjr2FJ8NUTm9i3ZmtHcS0vQU5xQQJjO2X4HQUc4yO69GJq8f05t/LtrIsv9TpOMaYFozvn8j+mnqWb97jdBQToEr3VXP/7GyGpcRytU1C5VUOW4yLyC9FZA0wQERWN1o2AavbL6JpLQtySxieGkdcVJjTUUwruOWM/qR2juSOd9fYhCLmmIlIpojMEpHv3O/1a0TE3utbyei+8YQGC4s27HI6iglQD3ywloqqWv54/jBCbPZtr9Lc38Z/gHOA2e4/Dy4jVfXydshmWlHpvmpWby9jwgDrouIvosJCePS8oWzaVcmfv9jgdBzj+/4N/BP4EQ3v9We7/zStIDo8hON6dGJxnt3Eadrf/JxiZq8q5LpT+jGgqzVw8DaHLcZVtUxVN6vqJaq6BThAQ3vDDiLSw5ODi8hkEckRkTwRuaOJ58NF5C3388tEpFej5+50r88RkUktHVMaPCwiua4bK6kAACAASURBVCKyTkRu8OgnECAWbdiFasOlUuM/xqYlcGFmCi8uzCdre5nTcYxvK1HV2aq6SVW3HFycDuVPxvVLIGt7uc0TYNrV/po67pmVRd/EaH45oa/TcUwTPLmB8xwR2QBsAhYAm4GPPdgvGHiWhvHl6cAlIpJ+yGbXAHtUtR/wFPAH977pwMXAYGAy8JyIBLdwzKuAVGCgqg4C3mwpYyBZkFtC5+gwMpJjnY5iWtndU9LpHB3Gb2euprbe5XQc47vuF5F/iMglIjL94OJ0KH8yzn0y5KuNdp+HaT9Pf76B7XsP8Oj0oYSHBDsdxzTBk0FDvwdOBHJVtTcwEVjqwX6jgDxVzVfVGhqK42mHbDMNeMX9eCYwURp67k0D3lTValXdBOS5j9fcMX8JPKiqLgBVtR5Sbi6XsjC3hJPTEqyNkR+KjQrloWmDWbujnL8vync6jvFdPwGG03AC5OCwxLMdTeRnMpJjiY0MZZH1GzftJLuwjJcWb+Li41MZ1buz03HMYYR4sE2tqpaKSJCIBKnqfBF52oP9koFtjb4uAE443DaqWiciZUC8e/3SQ/ZNdj8+3DH7AheJyHlACXCDqv7PQFoRuRa4FqBHD49G2/i8rMIySitrfph4wvifyUO6MWlwEn/+fANnZXSjZ7y1rDJH7HhVHeB0CH8WHCSM6RfP4rxdqKrN92DaVL2rYcr7TlFh3HnmIKfjmGZ4cmZ8r4h0ABYC/xaRPwOVbRvrqIQDVaqaCfwdeLmpjVT1RVXNVNXMxMTAKE4X5JQgAienBcbrDVQPTB1CaHAQd8/KQlWdjmN8z9dNDCU0rWxsv0R2lFWxscQbP0aNP3l1yWZWF5Rx3znpxEaFOh3HNMOTYnwasB+4CfgE2Ihnd9hvp2EM90Ep7nVNbiMiIUAsUNrMvs0dswB41/14FjDUg4wB4cvcEjKSY4nvEO50FNOGusZGcNukASzO28V7Kw/9r2ZMi04EVrpvkLfWhm1kXFrDPA+LNthQFdN2Cvce4PG5OYzvn8g5Q7s5Hce0oMViXFUrVdWlqnWq+grwVxrGFLbkWyBNRHqLSBgNN2TOPmSb2cCV7sfnA/O04ZTebOBid7eV3kAa8E0Lx3wPOMX9eDyQ60FGv1e2v5bvt+6xLioB4vITezI8NY6HPlzHnsoap+MY3zKZhvfaM7DWhm0mtXMUveKjWGz9xk0bUVXuez+belV+f+4QGw7lA5qb9Keju73gX0XkDHfrwOuBfODClg6sqnXA9cBcYB0wQ1WzReRBEZnq3uwlIF5E8oCbgTvc+2YDM4C1NJyNv05V6w93TPexHgN+5J6o6FHgp0f2o/BPi/N24VKYYOPFA0JwkPDo9AzKD9TyyJx1TscxvkUPs5hWNjYtgaX5pdTUWfcj0/rmZhfx+bqd3HRaf1I7Rzkdx3iguRs4XwP2AEtoKGzvAgQ4V1VXenJwVZ0DzDlk3X2NHlcBFxxm34eBhz05pnv9XuAsT3IFki9ziukYEcKwlDino5h2MqhbR346rg9/W7CR6SNSGN033ulIxjd8REPxLUAE0BvIoaHFrGlF49ISeX3pVr7fuocT+tj/T9N6yqtquX92NoO6deTqsTblva9obphKH1W9SlVfAC6hoa/3JE8LceM8VWVBbgnj0hJt6tsAc+PENFI7R3L3rDVU1dY7Hcf4AFXNUNWh7j/TaGglu8TpXP5odN94goOExXk2VMW0rsfn5lBcUc2j0zMItc99n9Hc31TtwQeqWg8UuM9kGx+xvqiC4opqa2kYgCLDgnn43Azyd1Xy3JcbnY5jfJCqfsf/tqM1raBjRCjDUmJZaOPGTSv6buseXlu6hStH92J4ql0N9yXNDVMZJiLl7scCRLq/FkBVtWObpzPH5Muchrv17ebNwHRy/0SmDe/O81/mMXVYN/p1iXE6kvFiInJzoy+DgBFAoUNx/N64tET+Mm8DZftrre2cOWa19S7uencNSTER3HJGf6fjmCN02DPjqhqsqh3dS4yqhjR6bIW4D1iQW8zArjEkdYxwOopxyL1npxMVFsJd72bhctm9eKZZMY2WcBrGkB86a7JpJePSEnApfL3Rzo6bY/ePRZtYX1TBA9MGExNhv9z5GhtQ5Kf2VdexfPMeJgzo4nQU46CEDuHcNWUg32zezYzl21rewQSytar6gHt5WFX/jbU2bDPDUuOICQ+xoSrmmG0t3c+fv8jljPQkJg3u6nQccxSsGPdTX+Xtos6lNkTFcGFmKqN6d+aROesoqah2Oo7xXnd6uM60gtDgIE7sG8/iPJv8xxw9VeXu99YQEhTEA9Os8ZGvsmLcTy3ILSE6LJiRPTs5HcU4TER45LwMqmpdPPThWqfjGC8jImeKyF+AZBF5ptHyL6DO4Xh+bVxaAtt2H2BLaaXTUYyPmr2qkEUbdnHrGf3pFhvpdBxzlKwY90OqyoKcEsb0SyAsxP6KDfTr0oFfndKX2asK+TKn2Ok4xrsUAsuBKmBFo2U2MMnBXH5vXFrDlUsbqmKOxt79NTz4wVqGpcZxxeheTscxx6DFSk1EKkSk/JBlm4jMEpE+7RHSHJmNJfvYvveAtTQ0/+WXE/rSNzGae97LYn+NnfA0DVR1laq+AvRT1VcaLe+q6h6n8/mzXvFRJMdFsniDDVUxR+7ROevZe6CWR8/LIDjIprz3ZZ6cNn0auA1IBlKAW4H/AG8CL7ddNHO0rKWhaUp4SDCPnJdBwZ4D/PnzDU7HMd5nlIh8JiK5IpIvIptEJN/pUP5MRDi5fwJf55VSV+9yOo7xIcvyS3lr+TZ+OrY36d2twZ2v86QYn6qqL6hqhaqWq+qLNMzE+RZgA5K90ILcEvomRpPSKcrpKMbLnNAnnosyU/nH4k1kF5Y5Hcd4l5eAJ4GxwPFApvtP04bG9kukorqOVQX2/9F4prqunjtnrSGlUyQ3npbmdBzTCjwpxveLyIUiEuReLqRhbCGANS72Mgdq6lm2abe1NDSHdeeUgXSKCuWud9dQb73Hzf8rU9WPVbVYVUsPLi3tJCKTRSRHRPJE5I4mnu8pIl+IyGoR+VJEUho9d6WIbHAvV7b2C/IFJ/WNRwQW27hx46Hnv9xIfkklvz93CFFhzc3daHyFJ8X4ZcAVQDGw0/34chGJBK5vw2zmKCzdVEpNncuGqJjDiosK496z01lVUMZrSzY7Hcd4j/ki8icRGS0iIw4uze0gIsHAs8CZQDpwiYikH7LZ48CrqjoUeBB41L1vZ+B+4ARgFHC/iATc1dZO0WEMTY5lkY0bNx7IK97Hc/M3cs6w7nbSzY+0+CuVquZz+IkfFrduHHOsFuSUEBEaxKjenZ2OYrzY1GHdmbmigD/NzWHSkK7WEstAQ1EMDcNTDlLg1Gb2GQXkuT8nEJE3aZi1s3EPzXTgZvfj+cB77seTgM9Udbd738+AycAbx/AafNLYtAT+tiCfiqpamz3RHJaqcvesNUSEBnHf2Yf+zmt8mSfdVBJF5C4ReVFEXj64tEc4c+QW5pZwYp94IkKDnY5ivJiI8PC5GdSrcv/72U7HMV5AVU9pYmmuEIeGG/sbT+1a4F7X2CpguvvxeUCMiMR7uC8icq2ILBeR5SUl/nn2eGy/ROpdytL83U5HMV7s7eUFLNu0m7umDCIxJtzpOKYVeTJM5X0gFvgc+KjRYrzM1tL95O+qtCEqxiM94qO4cWJ/Pl27k7nZRU7HMQ4TkSQReUlEPnZ/nS4i17TCoW8FxovI98B4YDtQ7+nOqvqiqmaqamZion++t43oGUdUWLANVTGHtWtfNQ/PWceoXp25MDPV6TimlXlSjEep6u2qOkNV3zm4tHkyc8QWuN/IT7Zi3Hjop+N6M7BrDPe/n01FVa3TcYyz/gXMBbq7v84FftPCPtuBxpVBinvdD1S1UFWnq+pxwN3udXs92TdQhIcEc0LvznYTpzmshz5cy/6aOh6ZPoQg6ynudzwpxj8UkSltnsQcswU5JaR0iqRPQrTTUYyPCA0O4tHpGeysqOKJT3OdjmOclaCqMwAXgKrW0fIZ7G+BNBHpLSJhwMU0zNz5AxFJEJGDnzV38v/zU8wFzhCRTu4bN89wrwtIY9MSyd9VScGe/U5HMV5mQW4J768s5JcT+tGvS4zTcUwb8KQYv5GGgvyAe/bNChEpb+tg5sjU1LlYsnEX4/snImK/NRvPHdejE1ec2JNXlmxm5ba9Tscxzql0j+VWABE5EWi2+bW7YL+ehiJ6HTBDVbNF5EERmerebAKQIyK5QBLwsHvf3cBDNBT03wIPHryZMxCdnJYAWItD898O1NRz73tZ9EmI5lcT+jodx7QRT7qp2K9hPmDFlj1U1tTbeHFzVG6bNIC52UXc+e4aZl8/htBgT35PN37mZhrOavcVka+AROD8lnZS1TnAnEPW3dfo8Uxg5mH2fRmbyRmAfl06kNQxnEV5u7h4VA+n4xgv8ecvNrB1937e+NmJ1pjBjx32E1dEBrr/HNHU0n4RjScW5JYQEiSc1C/B6SjGB8VEhPLA1MGs21HOy4s3OR3HOEBVv6PhBsuTgJ8Dg1V1tbOpAoeIMLZfIl/l7bLJuAwA63aU8/dF+VwwMoXRfeOdjmPaUHNnxm8GrgWeaOK5lnrPmna2ILeEzF6d6BBus3GZozNpcFdOG5TEU5/nMiWjG6mdo5yOZNqRewKfKUAvGj4bzhARVPVJR4MFkJP7J/DOdwVkF5YxNCXO6TjGQfUu5c531xAbGcpdUwY5Hce0scOeGVfVa91/Hk3vWdOOdpZXsW5HOeP722xc5uiJCA9OG0ywCPe8l4WqnZ0LMB8AVwHxQEyjxbSTMe4rm4ts3HjA+/eyLazctpf7zk6nU3SY03FMG/PoNKqInMT/ny0BQFVfbaNM5ggtzG1oaWjjxc2x6h4XyS1nDODBD9fyweodTB3WveWdjL9IcU9ZbxyS0CGc9G4dWbShhOtO6ed0HOOQorIq/vhJDuPSEpg23N6DA4EnM3C+BjwOjAWOdy+Zze5k2tWC3BISY8IZ1M1OYpljd+VJvRiaEsuDH2RTtt96jweQj0XkDKdDBLpxaQms2LKH/TV1TkcxDrl/dha19S5+f+4Q644WIDxpmZAJjFHVX6nqr93LDW0dzHim3qUs2mAtDU3rCQ4SHjkvgz37a3nsk3VOxzHtZykwy9rYOmtsWgK19cqy/IDt8hjQPs0uYm72Tm48LY2e8TZnSKDwpBjPArq2dRBzdFYV7KXsQK0NUTGtakhyLFeP6cUb32zjm01WFASIJ4HRNMy63FFVY1S1o9OhAs3xvToTHhJk48YD0L7qOu6fnc3ArjH8bFwfp+OYduRJMZ4ArBWRuSIy++DS1sGMZxbklBAkMNZaGppWdtPp/UmOi+SuWWuormtpIkbjB7YBWWp37joqIjSYUb07szivxOkopp09PjeHovIqHpmeYXM9BBhPbuD8XVuHMEdvQW4Jw1Lj7G5r0+qiwkL4/blD+Mm/vuWFBfncMDHN6UimbeUDX4rIx0D1wZXW2rD9je2XwKMfr6eorIqusRFOxzHtYNW2vbyyZDNXnNiTET06OR3HtLNmf/Vy9539naouOHRpp3ymGXsqa1hVsNeGqJg2c8rALpw1tBt/nZ9Hfsk+p+OYtrUJ+AIIw1obOmpcWsN7+uI8G6oSCOrqXdz57hq6xIRz26QBTscxDmj2zLiq1ouIS0RiVbWsvUIZzyzK24WqtTQ0bev+c9JZmFvC3bOy+M/PTrAbhf2Uqj4AICId3F/bb18OGdg1hoQOYSzeUML5I1OcjmPa2MtfbWLtjnL+dvkIYiJCnY5jHODJoKR9wBoReUlEnjm4eHJwEZksIjkikicidzTxfLiIvOV+fpmI9Gr03J3u9TkiMukIjvmMiATEh8iCnBLiokJtpjbTprrERHDHmQNZkl/KzBUFTscxbUREhojI90A2kC0iK0RksNO5AlFQkDCmXwKL83bhctkQfn+2bfd+nvwsl9MGJTFpsPXKCFSeFOPvAvcCC4EVjZZmuYe4PAucCaQDl4hI+iGbXQPsUdV+wFPAH9z7pgMXA4OBycBzIhLc0jFFJBMIiMFWLpeyILeEsf0SCA6yM5WmbV1yfA8ye3bi4TnrKN1X3fIOxhe9CNysqj1VtSdwC/B3hzMFrHFpiezaV8P6ogqno5g2oqrc814Wwe7Zj+2qY+BqsRhX1VeaWjw49iggT1XzVbUGeBOYdsg204CDx5oJTJSGf43TgDdVtVpVNwF57uMd9pjuQv1PwG89yObzsgrL2LWvmlMHdnE6igkAQUHCI9MzqKyu4/cfWe9xPxWtqvMPfqGqXwLW6NghBztkLdpgXVX81Qerd7Agt4RbJw2ge1yk03GMgzyZgTNNRGaKyFoRyT+4eHDsZBpaZR1U4F7X5DaqWgeUAfHN7NvcMa8HZqvqjhZez7UislxElpeU+O6b3Lz1xYjAhAFWjJv20T8php+f3JdZ32+3AsE/5YvIvSLSy73cQ0OHFeOArrERpHXpYDdx+qm9+2t48IO1DEuJ5cejezkdxzjMk2Eq/wSeB+qAU4BXgdfbMtSREpHuwAXAX1raVlVfVNVMVc1MTPTdGx/nry/muNQ4OltLQ9OOrj+1H73io7jnvSyqaq33uJ+5GkikYWjiu+7HVzuaKMCNS0vkm0277f+aH/r9R+vYu7+GR6Zn2FBT41ExHqmqXwCiqltU9XfAWR7stx1IbfR1intdk9uISAgQC5Q2s+/h1h8H9APyRGQzECUieR5k9EklFdWsKiizISqm3UWEBvPweRlsKd3PM19scDqOaUWqukdVb1DVEe7lRlXd43SuQDYuLYHqOhffbrZZcP3JwtwSZq4o4Bfj+zK4e6zTcYwX8GTSn2oRCQI2iMj1NBS/HTzY71sgTUR6u/e5GLj0kG1mA1cCS4DzgXmqqu4ZPv8jIk8C3YE04BtAmjqmqmYDP9yGLCL73DeF+qUvc4qBhh7QxrS3Mf0SmD4imRcX5nP20O6kd7cZ031ZSzMqq+rU9spi/tsJfToTGiws3rDrh97jxrdVVtdx57tr6JMYzfWn+m2ZYo6QJ8X4jUAUcAPwEA1DVa5saSdVrXMX73OBYOBlVc0WkQeB5ao6G3gJeM19Fns3DcU17u1mAGtpGB5znarWAzR1zCN5wf5g3vpiunaMIL2bFUHGGfeclc6CnBJuf2c1s351EiE2dbMvG03DvThvAMtoOOlhvEBUWAgje3Zi4YZd3Ol0GNMq/jQ3h8KyA7z989FEhAY7Hcd4iRaLcVX9FkBEXKr6kyM5uKrOAeYcsu6+Ro+raBjr3dS+DwMPe3LMJrbx5My9T6qpc7Fowy7OGdbN2iAZx3SODuOBaYO5/j/f8/dFm/jlhL5ORzJHrytwOnAJDVcvPwLeCMQTHd5oXFoif5qbQ0lFNYkx4U7HMcdgxZY9vLJkMz8+sSeZvTo7Hcd4EU+6qYwWkbXAevfXw0TkuTZPZpq0fPNu9lXXcYp1UTEOOyujG5MGJ/HU57lsLAmIebb8kqrWq+onqnolcCINrWS/dF+FNA4bl9bQ4vDrjdZVxZdV19Vz+zur6R4byW2TBzodx3gZT64tPw1MouHGSlR1FXByW4YyhzdvfTFhwUGMcfegNcYpIsJD04YQERLE7TNX20yBPsw9G/J0GjplXQc8A8xyNpUBGNw9lrioUBbmWjHuy56dl0de8T4ePm8IHcI9GSFsAolHAz1Vddshq6zPkkPm5RRzQp/ORNt/ZuMFunSM4L5zBrN8yx5eXbLZ6TjmKIjIqzTcRD8CeEBVj1fVh1T10O5XxgHBQcKYvgkszitB1X7h9UXrdpTz3JcbmX5css0NYprkSTG+TUROAlREQkXkVsCm4HPAltJK8ksqraWh8So/GpHM+P6J/HFuDtt273c6jjlyl9PQsepG4GsRKXcvFSJS7nA2Q8NQlZ3l1WwotuFgvqau3sXt76wmNjKUe89OdzqO8VKeFOO/oOGyZTIN7QSHA79qy1CmafPWN7Q0tGLceBMR4ZHpGQhw57tr7Oydj1HVIFWNcS8dGy0xqmotm7zAyf0b2houyLGZb33Ny19tYnVBGQ9MG0wnm6TPHEaLxbiq7lLVy1Q1SVW7qOrlwI/bIZs5xLz1xfRJjKZnfLTTUYz5L8lxkdwxZRCL83YxY/mho9qMMceie1wkA5JimO+eY8L4hs27Knnys1xOT0/irIxuTscxXuxomwPf3KopTIv2VdexLH83p9p4M+OlLhvVg1G9O/P7j9axs7zK6TjG+JUJAxP5dvNuKqpqnY5iPOByKbfNXEVocBAPTRtirYhNs462GLd/Ve1sQU4JNfUuTk9PcjqKMU0KChL++KOh1Na7uHuWDVcxpjWdMqALtfXKV3mlTkcxHvjn15v5dvMe7j9nMF1jI5yOY7zc0Rbj9inbzj5bW0Tn6DBG9uzkdBRjDqtXQjS3nD6Az9cVM3tVodNxjPEbI3t2IiY8hC9tqIrXyy/Zxx8/Wc9pg7rwoxHJTscxPuCwxfjBO+mbWCqA7u2YMeDV1ruYt76YUwd2sWnHjde7emxvhqfGcf/sbIptuIoxrSI0OIhx/ROYn1NsV528WL1LufXtVUSEBvPIeRk2PMV45LCVXRN31je+w96aXLejbzbtpryqjjNsiIrxAcFBwuMXDONATT13WHcVvycik0UkR0TyROSOJp7vISLzReR7EVktIlPc60NF5BURWSMi60TkzvZP71smDOjCzvJq1u2ocDqKOYx/LMrnu617eXDaYLp0tOEpxjN2mtUHfJpdRERoEOPSEp2OYoxH+nXpwO2TBzJvfTFvLy9wOo5pIyISDDwLnAmkA5eIyKHNlO8BZqjqccDFwHPu9RcA4aqaAYwEfi4ivdojt6+a4G5xaF1VvNOGnRU88VkukwYnMXWYDSAwnrNi3MupKp+t3cnYfolEhgU7HccYj111Ui9O7NOZBz9ca5MB+a9RQJ6q5qtqDfAmMO2QbRQ42K88FihstD5aREKASKAGsEmGmtGlYwRDkjvauHEvVFfv4ta3VxEdFszvz7XhKebIWDHu5bILyyksq+KMwTZExfiWoCDhT+cPA+C3M1fjctlwFT+UDDRuLF/gXtfY74DLRaQAmAP82r1+JlAJ7AC2Ao+r6u42TesHThnQhRVb9lC231ocepMXFuazqqCMh84dQmJMuNNxjI+xYtzLfbp2J0ECE23WTeODUjtHce/Zg1iSX8qrSzY7Hcc44xLgX6qaAkwBXhORIBrOqtfT0BCgN3CLiPQ5dGcRuVZElovI8pISm4FywoBEXAqL8uxn4S3WF5Xz9Oe5nJXRjbOH2vAUc+SsGPdyn63dSWbPzsR3sN+0jW+6MDOVUwd24bFP1pNfss/pOKZ1bQdSG32d4l7X2DXADABVXQJEAAnApcAnqlqrqsXAV0Dmod9AVV9U1UxVzUxMtPtmhqd2Ii4qlPnrrRj3BtV19dz81io6RoTy4LTBTscxPsqKcS+2bfd+1u0ot4l+jE8TER6bnkF4SDC3vL2KunqX05FM6/kWSBOR3iISRsMNmrMP2WYrMBFARAbRUIyXuNef6l4fDZwIrG+n3D4rOEg4OS2RBbnFNvTLCzz5WS5rd5Tz2I+G2kkzc9SsGPdic7OLAKwYNz6vS8cIHjp3CN9v3csLC/OdjmNaiarWAdcDc4F1NHRNyRaRB0VkqnuzW4Cficgq4A3gKm3od/ks0EFEsmko6v+pqqvb/1X4nlMGJrJrXw1rtpc5HSWgLc0v5cWF+Vwyqod9TptjYv3CvdhHa3aQ3q0jvRKinY5izDGbOqw7c7OLePrzXE5OSyQjJdbpSKYVqOocGm7MbLzuvkaP1wJjmthvHw3tDc0RGt+/C0ECX6zbybDUOKfjBKSyA7XcMmMVveKjuffsQU7HMT7Ozox7qcK9B/h+617OGtrN6SjGtJqHzx1CQodwbnzze/bX1Dkdxxif1Dk6jMyenfl07U6nowSs+9/Poqi8iqcuGk5UmJ3XNMfGinEvNWfNDgCmZFgxbvxHXFQYT144nE2llTz04Vqn4xjjs05PT2J9UYX18HfA+yu3897KQm6cmMZwuzJhWoEV415qzpodDOrWkd42RMX4mdF94/nF+L688c02Psna4XQcY3zSwTHKn9nZ8XZVuPcA97yXxXE94vjVhL5OxzF+wopxL1S49wDfbd3LWRldnY5iTJu46bT+DE2J5fZ31rCj7IDTcYzxOb0Soumf1MGK8Xbkcim3zFhFvUt5+qLhhARbCWVah/1L8kIfZzV0UbEhKsZfhYUE8eeLj6O23sXNbzV8uBljjszp6Ul8s3k3e/fXOB0lIDy/YCNL8ku5/5x0esbbVWvTeqwY90IHh6j0SezgdBRj2kzvhGh+d85glrjbgxljjszp6V2pdynzc4qdjuL3lm/ezZOf5XL20G5cmJna8g7GHAErxr3MjrIDrNiyx4aomIBwQWYKUzK68sSnOazattfpOMb4lKHJsXSJCbehKm1s7/4abnjje5LjInlkegYi4nQk42esGPcyH622LiomcIgIj543lC4x4fz6je8pr6p1OpIxPiMoSDgtPYkFOSVU19U7HccvqSq/nbmakn3V/OWS4+gYEep0JOOHrBj3MrO+386wlFgbomICRmxUKH+59DgK9x7gt2+vpmFyRmOMJ05PT6Kypp6vN5Y6HcUvvfL1Zj5du5PbJw+0CZZMm7Fi3Ivk7qwgu7Ccc49LdjqKMe1qZM/O3D55IJ9kF/HPrzY7HccYn3FS33iiw4L5NNuGqrS2rO1lPPJ/7d15fFTV/f/x1yeThJCwBJKwhiXsgoBgRFGxuFQRF6rVr+AuIlq3fq1dXPrQ1sf3+6u1/txqLaUgYqUg7lRR6r4bCDthjSyGPYAEAmY/3z/mokOaQBIycyfJ+/l4zCN3zr33i0uOgAAAHShJREFU3M+czJx8cuece+eu5qx+7bjx9Ay/w5FGTMl4FHl98RYCMcaFgzr5HYpIxE0YkcE5x7XnD2+vYonGj4vUSLPYAGf2a8e/c7ZTVl7hdziNRmFxGXfMXEzbpHgevXywxolLWIU1GTezUWa2xsxyzeyeKtY3M7MXvfVZZtY9ZN29XvkaMzvvaHWa2QyvfIWZPWtmDWpgV0WF440lWxnRO5W0ls38Dkck4syM/3/5YNq3SuC2GYt0uTaRGrpgYEd2Hyhh/oY9fofSKDjn+M3Ly9i0+wBPjD2BtknxfockjVzYknEzCwB/Ac4H+gPjzKx/pc1uBL51zvUCHgf+6O3bHxgLDABGAc+YWeAodc4A+gEDgebAhHC9tnCYv3EPW/Z+xyUaoiJNWOvEOP5y5VB27i/ilxo/LlIjI/u2IzE+wJvLdUfb+jD1sw28tXwbvx7Vj1N6pPgdjjQB4TwzPgzIdc6td86VALOAMZW2GQNM95ZfBs624HdBY4BZzrli59wGINerr9o6nXNznQeYD6SH8bXVu9cXbyExPvD9LY5FmqrBXZK5b/RxvLdqB3//VNcfFzma5vEBzj6uPe+s0FCVY5W1fjd/eHs15w1oz81n9PA7HGkiwpmMdwbyQp5v9sqq3MY5VwYUAClH2PeodXrDU64B3qkqKDObaGbZZpadn59fy5cUHkWl5by1fBujBnQgMT7W73BEfHf9qd0ZPbADD7+9ms/W7fI7HJGod8HADuw5UEKWhqrU2Y59Rdz2z8V0a5uoceISUY1xAuczwCfOuU+rWumcm+ycy3TOZaalpUU4tKq9s2I7+4vKuOzEBnUyXyRszIw/XTaYXu1acPvMReTtOeh3SCJR7fuhKss0VKUuSssruG3GIg6WlDHpmhNpqeuJSwSFMxnfAoTeMzbdK6tyGzOLBVoDu4+w7xHrNLMHgTTgF/XyCiJk5vxv6JaSqLFpIiGSmsUy+ZpMKiocE/+xkO9KdFMTkeokxAU457j2zNNVVerk/81dRfamb/njTwfRp31Lv8ORJiacyfgCoLeZZZhZPMEJmXMqbTMHuM5bvgz4wBvzPQcY611tJQPoTXAceLV1mtkE4DxgnHOuwfRE6/MLydqwh//K7EJMjL4SEwnVPTWJp8YNYfX2ffz6FU3oFDmS0QM7sudACV+t11CV2ngpO49pn29k/GkZXDRYlxaWyAtbMu6NAb8dmAesAmY753LM7CEzu9jbbCqQYma5BM9m3+PtmwPMBlYSHPt9m3OuvLo6vbomAe2BL81siZk9EK7XVp9ezM4jEGNcriEqIlUa2bcdvzy3L/9aulUTOkWOYGTfNJLiA/xr6Va/Q2kwsjfu4f7XVnB6r1TuG93P73CkiQrrbEHn3FxgbqWyB0KWi4DLq9n3f4H/rUmdXnmDm/lYWl7BKws3c1a/drRrleB3OCJR69aRPcnZWsDDb6+mb4dW/KhPdMz3EIkmCXEBzju+A3NXbOP3YwaQEBfwO6Sotvnbg9zywkI6t2nOX64cSmygMU6jk4ZA7zwfvb9qB7sKSxg3rMvRNxZpwg5N6OzTviW3z1jEmu37/Q5JJCpdOiSd/UVlvL9qp9+hRLUDxWVMmJ5NcVkFf782k9aJmrAp/lEy7qPpX2yiU+sEzuits3wiR5PULJap159EQnyA8c8tIH9/sd8hiUSd4T1T6NAqgVcXbfY7lKhVUeG468UlrN2xn6evHEqvdi38DkmaOCXjPlm1bR9frt/Ntad211djIjXUObk5U6/LZPeBYiY8n01Rqa6wIhIqEGOMGdKJj9bms6tQ/7BW5ZF5a/j3yh389oL+GvImUUFZoE+mf7GRhLgYxp6kISoitTEoPZknxw5h2ea9/GL2EioqdIUVkVCXDkmnvMJpImcVnv9yI5M+/pqrTu7KDad19zscEUDJuC/2HCjhtcVbuGRIOsmJ8X6HI9LgnDegA/edfxxzl2/nkXlr/A5HJKr07dCSAZ1a8eqiyrf2aNrm5WznwTk5nHNcex4ac7zusClRQ8m4D2bO/4bisgr9Vy5yDCaMyODKk7sy6eOvefazDX6HIxJVLh2azvItBazdocnOAAs3fcudMxczOD2ZP48bQkD39ZAoomQ8wopKy3nui42M6J2qu3yJHAMz46GLB3DegPY89OZKXlusCWsih4w5oRNxAWPW/Dy/Q/Hd+vxCJkxfQMfWCUy9LpPm8brko0QXJeMR9lJ2Hvn7i/nZyJ5+hyLS4MUGYnhy7BCG90jhVy8t48PVupybCEBqi2acO6ADryza3KQnOm/d+x3XTJ1PjBnTxw8jpUUzv0MS+Q9KxiOotLyCSR+vZ2jXZIb3SPE7HJFGISEuwORrT6Rfx5b8bMZCFm7SrcBFAK4a1pWC70qZu3yb36H4In9/MVdPyWLfd6VMHz+MbilJfockUiUl4xH0+uItbNn7Hbef1UsTR0TqUcuEOJ67YRgdWzfnhmkLWLGlwO+QRHw3vGcKPVKTmJH1jd+hRNzegyVcMzWLbQVFPDf+JI7v3NrvkESqpWQ8QkrLK3jmo6/p37EVZ/Zt53c4Io1OaotmPD9+GC0T4rh6ahY5W5WQS9NmZowb1pWFm75tUnetLSwu47ppC1i/6wBTrsvkxG5t/Q5J5IiUjEfIrAV5bNh1gLvP7aOz4iJh0qVtIjNvOoXEuABXT8li1bZ9fofU6JnZKDNbY2a5ZnZPFeu7mtmHZrbYzJaZ2eiQdYPM7EszyzGz5WaWENnoG7+fnphOfCCGGVmb/A4lIvYXlXLDtPnkbCngmSuHclqvVL9DEjkqJeMRcKC4jCffW8ewjLac1U9nxUXCqWtKIjMnnkJCXICrpmSxersS8nAxswDwF+B8oD8wzsz6V9rst8Bs59wQYCzwjLdvLPACcItzbgAwEiiNUOhNRtukeC4c1JFXFm6m4LvG3bwF35VyzdT5LP5mL0+NG8I5/dv7HZJIjSgZj4Apn25gV2Ex95zfT2fFRSKgW0oSM286hfhADFf+PUtjyMNnGJDrnFvvnCsBZgFjKm3jgFbecmvg0G0hzwWWOeeWAjjndjvnmu5lP8LoxhEZHCgpZ+b8xjt2fO/BEq6eEhye9sxVQxk9sKPfIYnUmJLxMMvbc5C/fpzL6IEdGNq1jd/hiDQZ3VOTmDnxFJrHBRg3+Svmb9BVVsKgMxB6IevNXlmo3wFXm9lmYC5wh1feB3BmNs/MFpnZr6s6gJlNNLNsM8vOz8+v3+ibiAGdWnNarxSmfb6BkrIKv8Opd7sLixn39yzW7NjP5GsyOXdAB79DEqkVJeNh5Jzjd3NyiDHjtxdU/uZWRMItIzWJl24ZTrtWzbhmahYfrN7hd0hN0TjgOedcOjAa+IeZxQCxwOnAVd7PS8zs7Mo7O+cmO+cynXOZaWlpkYy7UZkwogc79hXz5rKtR9+4Acnbc5DLJn3J+vxCplybyZkaCioNkJLxMJqXs4P3V+/krnP60Cm5ud/hiDRJnZKbM/vm4fRp35KJzy/kjSVb/A6pMdkCdAl5nu6VhboRmA3gnPsSSABSCZ5F/8Q5t8s5d5DgWfOhYY+4iRrZJ43e7Vow+ZP1OOf8DqderNhSwCXPfMG3B0v4500nc0Yf/bMmDZOS8TDJ31/M/a8tp3/HVlx/Wne/wxFp0lJaNOOfN51MZvc2/HzWEp7+YF2jSUh8tgDobWYZZhZPcILmnErbfAOcDWBmxxFMxvOBecBAM0v0JnP+CFgZscibGDPj5h/1ZPX2/by7suF/Q/Tpunyu+NuXNIuN4eVbhuvyhdKgKRkPA+ccv3p5KYXFZTwx9gTiAmpmEb8dujHQT07oxKP/XsvdLy2luEzzBY+Fc64MuJ1gYr2K4FVTcszsITO72NvsbuAmM1sKzASud0HfAo8RTOiXAIucc29F/lU0HT85oRPdUxJ57N21VFQ03H9G/5n1DTdMW0CXtom8euup9GrX0u+QRI5JrN8BNEbPfPQ1H63J5/cXD6BPe3USItEiIS7A41ecQI+0Fjz27lry9hzkb9dk0jYp3u/QGizn3FyCQ0xCyx4IWV4JnFbNvi8QvLyhREBsIIafn9Obu15cyjs52xvcFUdKyyv4/b9yeOGrb/hRnzT+fOUQWiXE+R2WyDHTKdt6Ni9nO3+at4aLB3fi2uHd/A5HRCoxM+48uzd/HjeEpZsLuOjPn7Ekb6/fYYlExMWDO9MjLYnH311LeQM6O767sJirp2TxwlffcPMZPXj2+pOUiEujoWS8Hn2eu4s7Zy5mcJdkHrlskK4pLhLFLhrciZdvGQ7A5ZO+YPoXGzWOXBq9QIxx94/7sm5nIS8uyDv6DlFg/oY9XPDUZyzO28vjVwzm3tHHEYjR31dpPJSM15N3V+5g/HMLyEhN4tnrMkmIC/gdkogcxaD0ZN6683RG9E7jwTk53DFzMfuKGvddCkVGD+zAsO5tefTfayg4GL3v9/IKx1Pvr2Ps5C9JiIvh1Z+dyiVD0v0OS6TeKRk/RsVl5Tw6bw03PZ9N3w4tmTHhZFJaNPM7LBGpoeTEeKZcm8mvzuvL2yu2M+rxT/g8d5ffYYmEjZnx4MX92XuwhCfeX+t3OFXa/O1Brp6SxWPvruWiwZ14884RHN+5td9hiYSFJnDW0YHiMt5avo1JH33N+l0HuOzEdP7nJ8frjLhIAxQTY9x2Zi9O7ZnC3S8t5aopWVw3vBv3nH8czeP1mZbGZ0Cn1owd1pXnv9zEJUM6Myg92e+QAKiocMyY/w0Pz10FwCOXDeLyE9M17FMaNSXjtbCrsJj/nrWEXYXF5O4spKzC0bd9S6aPH8aPdLMBkQZvSNc2vHXHCB6Zt5ppn2/k/dU7efCiAfy4f3u/QxOpd78Z1Y8PVu3k7tlL+dcdp/t+Munr/ELuf205X63fw4jeqfzh0oGkt0n0NSaRSFAyXgsJcQEOlJTRpW0iZ/Zrx8g+aQzLaKv/2EUakebxAR68aACjBnTgt6+v4Kbnszm7XzsevGgAXVOUGEjj0bp5HA//dCDXT1vAY++u5b7Rx/kSx76iUp56bx3PfbGR5nEBHr50IFec1EV/W6XJUDJeCy2axfLarVVeLldEGpmTe6Qw9+cjmPb5Bp54bx3nPPYxV57cldvO7EVaS80LkcZhZN92XHVyVyZ/sp6hXZMZdXzkrj1eXFbO7OzNPPneWnYfKOGKzC7cfW5ffb6kyVEyLiJSjbhADBPP6MlFgzvx5Hvr+MdXm5idnccNp3XnhtMySNVkbWkEHrioPzlb93H37KVkpLagb4fw3qyupKyC2dl5PPNhLlsLijipexumXT+MgemaoClNkzXl6+pmZma67Oxsv8MQkQbi6/xCHn93LW8u20Z8bAyXDunMjadn0NuHO+2a2ULnXGbED+wj9dnhs72giIuf/gyAF28eTkZqUr0fY+f+ImbNz2NG1iZ27CtmSNdk7jqnDyN6p2pIijR6R+qzlYyrYxeRWsrdWcizn2/glYWbKS6rILNbGy4dms4FgzrSunlk7gqoZFzq27od+7li8lfEB2J49vqT6N+p1THXWVJWwcdr83ljyRbm5WyntNxxRp80JpyeoSRcmhTfknEzGwU8CQSAKc65hyutbwY8D5wI7AaucM5t9NbdC9wIlAN3OufmHalOM8sAZgEpwELgGudcyZHiU8cuIsdiz4ESXlyQxyuLNpO7s5D42BhO7ZnCmX3bcWbfdmGd8KlkXMJh1bZ9jH9uAXsPlvL7MQPqdFnB3YXFfJa7i0/W7uLdldvZV1RGm8Q4xpzQmWuGd6NnWoswRS8SvXxJxs0sAKwFfgxsBhYA45xzK0O2uRUY5Jy7xczGApc4564ws/7ATGAY0Al4D+jj7VZlnWY2G3jVOTfLzCYBS51zfz1SjOrYRaQ+OOdYvqWA1xdv5cM1O9mw6wAAHVolMLhLawalJ9OrXQu6pSTSpU0iSc2OfbqOknEJl537i7h9xmLmb9zDid3acNOIDM7ok0Zi/OHv26LScrYVFLF173es2b6fFVsLWLGlgLU7CoHg1VrO6teOiwd34vTeqcQFdJ9BabqO1GeHcwLnMCDXObfeC2IWMAZYGbLNGOB33vLLwNMW/Bd8DDDLOVcMbDCzXK8+qqrTzFYBZwFXettM9+o9YjIuIlIfzIxB6ckMSk/mgYv6s2HXAT5Zm8/ib75lSd5e5uXsOGz7+NgYWiXE0qJZLLGBGGbedIquICFRo13LBGZNPIUXs/N4+oNcbnlhEbExRvtWCcQFjIMl5XxXUs7+4rLD9ktr2YzjO7Xi4sGdGNE7jeM7tyYQo2EoIkcTzmS8M5AX8nwzcHJ12zjnysysgOAwk87AV5X27ewtV1VnCrDXOVdWxfaHMbOJwESArl271u4ViYjUQEZqEhmpSVx3ancACg6WsnH3ATbtOUjenoPs+66UfUVlFBaXUV5RQVxACYtEl5gYY9ywrlx+YjpffL2brA272VZQRFm5IzE+QPP4AG0S4+mc3JxOyc3pmZZEu1YJfoct0iA1uUsbOucmA5Mh+JWnz+GISBPQOjGOwYnJDO4SHbccF6mp2EAMZ/RJ4wzdZVokbMI5gGsL0CXkebpXVuU2ZhYLtCY4kbO6fasr3w0ke3VUdywRERERkagSzmR8AdDbzDLMLB4YC8yptM0c4Dpv+TLgAxecUToHGGtmzbyrpPQG5ldXp7fPh14deHW+EcbXJiIiIiJyzMI2TMUbA347MI/gZQifdc7lmNlDQLZzbg4wFfiHN0FzD8HkGm+72QQne5YBtznnygGqqtM75G+AWWb2P8Bir24RERERkailm/7oMlki0gDp0oYiIg3HkfpsXfRTRERERMQnSsZFRERERHyiZFxERERExCdKxkVEREREfNKkJ3CaWT6wqQ67pgK76jmcY6WYai4a41JMNReNcfkRUzfnXJO6E4v67IiIxrgUU81FY1yKKajaPrtJJ+N1ZWbZ0XYVA8VUc9EYl2KquWiMKxpjkh9E4+8nGmOC6IxLMdVcNMalmI5Ow1RERERERHyiZFxERERExCdKxutmst8BVEEx1Vw0xqWYai4a44rGmOQH0fj7icaYIDrjUkw1F41xKaaj0JhxERERERGf6My4iIiIiIhPlIyLiIiIiPhEyXgtmNkoM1tjZrlmdk8Y6u9iZh+a2UozyzGzn3vlbc3sXTNb5/1s45WbmT3lxbPMzIaG1HWdt/06M7supPxEM1vu7fOUmVkNYwuY2WIze9N7nmFmWV49L5pZvFfezHue663vHlLHvV75GjM7L6S8Tu1qZslm9rKZrTazVWY23O+2MrO7vN/dCjObaWYJfrSVmT1rZjvNbEVIWdjbprpjHCGmP3m/v2Vm9pqZJde1DerYzv8RU8i6u83MmVlqJNtJ6k9NPy/HUL/6bPXZ9dJWpj67Ru1cXVwh6xpmv+2c06MGDyAAfA30AOKBpUD/ej5GR2Cot9wSWAv0Bx4B7vHK7wH+6C2PBt4GDDgFyPLK2wLrvZ9tvOU23rr53rbm7Xt+DWP7BfBP4E3v+WxgrLc8CfiZt3wrMMlbHgu86C3399qsGZDhtWXgWNoVmA5M8JbjgWQ/2wroDGwAmoe00fV+tBVwBjAUWBFSFva2qe4YR4jpXCDWW/5jSEy1boPatnN1MXnlXYB5BG8wkxrJdtKj3vpT9dnqs9VnN7I+u7q4vPIG22/73mE2lAcwHJgX8vxe4N4wH/MN4MfAGqCjV9YRWOMt/w0YF7L9Gm/9OOBvIeV/88o6AqtDyg/b7ghxpAPvA2cBb3pv0F0hH8jv28b7IAz3lmO97axyex3arq7tCrQm2IlapXLf2opgx57nfbhjvbY6z6+2ArpzeCca9rap7hjVxVRp3SXAjKpe29HaoC7vySPFBLwMDAY28kOnHrF20uPYH7X9vNTTMdVnVx+T+mz12d9vV5f35NHiogH32xqmUnOHPrSHbPbKwsL7WmYIkAW0d85t81ZtB9ofJaYjlW+uovxongB+DVR4z1OAvc65sirq+f7Y3voCb/vaxno0GUA+MM2CX8VOMbMkfGwr59wW4FHgG2Abwde+EP/b6pBItE11x6iJ8QTPQtQlprq8J6tkZmOALc65pZVWRUs7Sc2oz1afrT67CfTZ0PD7bSXjUcjMWgCvAP/tnNsXus4F/yVzEYzlQmCnc25hpI5ZQ7EEv6b6q3NuCHCA4NdG3/OhrdoAYwj+0ekEJAGjInX82ohE29TmGGZ2P1AGzAhnTDWIIxG4D3ggUseM9PtU6p/67BpRn30M1GcfMZYG328rGa+5LQTHIx2S7pXVKzOLI9ipz3DOveoV7zCzjt76jsDOo8R0pPL0Wr6G04CLzWwjMIvg155PAslmFltFPd8f21vfGthdh1iPZjOw2TmX5T1/mWBH72dbnQNscM7lO+dKgVcJtp/fbXVIJNqmumNUy8yuBy4ErvI6uLrEtJvat3NVehL8w7zUe8+nA4vMrEMdYqrXdpJaU5+tPlt9duPvs6Ex9Nv1Nd6lsT8I/le/3vuFH5qEMKCej2HA88ATlcr/xOGTBh7xli/g8IkJ873ytgTH5rXxHhuAtt66yhMTRtcivpH8MBnoJQ6feHGrt3wbh0+8mO0tD+DwyR3rCU7sqHO7Ap8Cfb3l33nt5FtbAScDOUCit8904A6/2or/HH8Y9rap7hhHiGkUsBJIq7Rdrdugtu1cXUyV1m3kh7GHEWsnPeqlP1WfrT5bfXYj7LOriqvSuo00sH7b9w6zIT0IzspdS3Bm8P1hqP90gl97LAOWeI/RBMdKvQ+sA94LecMY8BcvnuVAZkhd44Fc73FDSHkmsMLb52kqTYo4Snwj+aFj7+G9YXO9D1QzrzzBe57rre8Rsv/93nHXEDLLva7tCpwAZHvt9br3gfK1rYDfA6u9/f5BsGOKeFsBMwmOgSwleEbqxki0TXXHOEJMuQTH7R16v0+qaxvUsZ3/I6ZK7biRHzr1iLSTHvX3qOnn5RjqV5+tPrte2gr12TVq5+riqrR+Iw2s3z50ABERERERiTCNGRcRERER8YmScRERERERnygZFxERERHxiZJxERERERGfKBkXEREREfGJknFp0sys0PvZ3cyurOe676v0/Iv6rF9EpKlRny2NkZJxkaDuQK069pA7h1XnsI7dOXdqLWMSEZGqdUd9tjQSSsZFgh4GRpjZEjO7y8wCZvYnM1tgZsvM7GYAMxtpZp+a2RyCdyHDzF43s4VmlmNmE72yh4HmXn0zvLJDZ3TMq3uFmS03sytC6v7IzF42s9VmNsPM7FB9ZrbSi+XRiLeOiEh0UZ8tjcbR/ksUaSruAX7pnLsQwOugC5xzJ5lZM+BzM/u3t+1Q4Hjn3Abv+Xjn3B4zaw4sMLNXnHP3mNntzrkTqjjWpQTvRDcYSPX2+cRbN4TgbYW3Ap8Dp5nZKuASoJ9zzplZcr2/ehGRhkV9tjQaOjMuUrVzgWvNbAmQRfA2uL29dfNDOnWAO81sKfAV0CVku+qcDsx0zpU753YAHwMnhdS92TlXQfBWw92BAqAImGpmlwIHj/nViYg0LuqzpcFSMi5SNQPucM6d4D0ynHOHzrIc+H4js5HAOcBw59xgYDGQcAzHLQ5ZLgdinXNlwDDgZeBC4J1jqF9EpDFSny0NlpJxkaD9QMuQ5/OAn5lZHICZ9TGzpCr2aw1865w7aGb9gFNC1pUe2r+ST4ErvDGOacAZwPzqAjOzFkBr59xc4C6CX5WKiDRl6rOl0dCYcZGgZUC599Xlc8CTBL9uXORNyMkHflLFfu8At3hjBNcQ/NrzkMnAMjNb5Jy7KqT8NWA4sBRwwK+dc9u9PwxVaQm8YWYJBM/+/KJuL1FEpNFQny2Nhjnn/I5BRERERKRJ0jAVERERERGfKBkXEREREfGJknEREREREZ8oGRcRERER8YmScRERERERnygZFxERERHxiZJxERERERGf/B/+ixctA3snIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot_lr(show_moms=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T19:09:40.594110Z",
     "start_time": "2019-07-25T19:09:40.253411Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEGCAYAAABLgMOSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXwV5b3H8c+PkIV9B4GAQbHsCOEUpdYFsbhUBZSqXKyiVap2ubdWW5dWapWWVq/a7bZ1KdTWYq0WsQpFsFhbK2hANgEFBJU9IJuELcnv/jGTeEhOIAw5OVm+79frvDLzzDNzfjMe+Z3nmec8Y+6OiIjIsWqQ6gBERKR2UgIREZFIlEBERCQSJRAREYlECURERCJpmOoAqlPbtm09Jycn1WGIiNQqCxYs2Obu7cqW16sEkpOTQ15eXqrDEBGpVczsg0Tl6sISEZFIlEBERCQSJRAREYmkXt0DSeTQoUOsX7+e/fv3pzqUOiMrK4vs7GzS09NTHYqIJFG9TyDr16+nWbNm5OTkYGapDqfWc3e2b9/O+vXr6datW6rDEZEkqvddWPv376dNmzZKHlXEzGjTpo1adCL1QL1PIICSRxXT9RSpH+p9F5akyKbFsOLFw8vKJR6r4u3JPHYSt1fpvhXtV9E+R3qvSuyTtGMdbVsVv3+jltDtrMTvUY8pgaTY9u3bGTZsGACbN28mLS2Ndu2CH3y++eabZGRkHPUY1113HXfccQc9evRIaqxVavMyeO2BuAI9l0ZqsE65MH5uqqOocZRAUqxNmzYsWrQIgB/84Ac0bdqU22677bA67o6706BB4h7HyZMnJz3OKjdwbPCqrLIPPiv3ILRj2H48+1b79qPtW/ZQld23outRyetU7ccqu3s1v396o4pjqceUQGqo1atXc+mllzJw4EDefvttZs+ezb333svChQvZt28fV155Jffccw8An//85/nlL39J3759adu2LTfddBMzZ86kcePGTJ8+nfbt26f4bKpA2S4K3WcRSbmUJRAzuw8YARQDW4Fx7r4xQb1rge+Fq/e7++/D8kHAFKARMAP4bz/O5/Pe+7d3WL5x9/EcopzenZoz4ZI+kfZduXIlTz75JLFYDIBJkybRunVrCgsLGTp0KKNHj6Z3796H7bNr1y7OPvtsJk2axK233srvfvc77rjjjuM+DxGRslI5CusBd+/v7gOAF4F7ylYws9bABOA0YDAwwcxahZt/DdwInBK+LqiWqKvRySefXJo8AKZOnUpubi65ubmsWLGC5cuXl9unUaNGXHjhhQAMGjSIdevWVVe4IlLPpKwF4u7xX/WbkPgu6vnAbHf/GMDMZgMXmNmrQHN3nxeWPwmMBGYeT0xRWwrJ0qRJk9LlVatW8bOf/Yw333yTli1bcvXVVyf8rUX8Tfe0tDQKCwurJVYRqX9S+jsQM5toZh8BY0nQAgE6Ax/Fra8PyzqHy2XLE73HeDPLM7O8/Pz8qgk8BXbv3k2zZs1o3rw5mzZtYtasWakOSUTquaQmEDObY2bLErxGALj73e7eBXgK+HoyYnD3R9095u6xkuGxtVFubi69e/emZ8+eXHPNNZxxxhmpDklE6jk7zvvOVROEWVdghrv3LVM+BjjH3b8arv8WeDV8zXX3nonqVSQWi3nZB0qtWLGCXr16VdGZSAldV5G6w8wWuHusbHnKurDM7JS41RHAygTVZgHDzaxVePN8ODDL3TcBu83sdAvmzbgGmJ70oEVEpFQqfwcyycx6EAzj/QC4CcDMYsBN7n6Du38cDvd9K9znhyU31IFb+HQY70yO8wa6iIgcm1SOwrq8gvI84Ia49d8Bv6ugXt+y5SIiUj00G6+IiESiBCIiIpEogYiISCRKICk2dOjQcj8KfOSRR7j55psr3Kdp06YAbNy4kdGjRyesc84551B2yHJZjzzyCAUFBaXrF110ETt37qxs6CJSzymBpNiYMWN4+umnDyt7+umnGTNmzFH37dSpE88++2zk9y6bQGbMmEHLli0jH09E6hclkBQbPXo0L730EgcPHgRg3bp1bNy4kYEDBzJs2DByc3Pp168f06eX/5nLunXr6Ns3GIi2b98+rrrqKnr16sWoUaPYt29fab2bb76ZWCxGnz59mDBhAgA///nP2bhxI0OHDmXo0KEA5OTksG3bNgAeeugh+vbtS9++fXnkkUdK369Xr17ceOON9OnTh+HDhx/2PiJSv+h5IPFm3gGbl1btMU/oBxdOqnBz69atGTx4MDNnzmTEiBE8/fTTXHHFFTRq1Ihp06bRvHlztm3bxumnn86ll15a4fPGf/3rX9O4cWNWrFjBkiVLyM3NLd02ceJEWrduTVFREcOGDWPJkiV885vf5KGHHmLu3Lm0bdv2sGMtWLCAyZMnM3/+fNyd0047jbPPPptWrVqxatUqpk6dymOPPcYVV1zBc889x9VXX10110pEahW1QGqA+G6sku4rd+euu+6if//+nHfeeWzYsIEtW7ZUeIzXXnut9B/y/v37079//9JtzzzzDLm5uQwcOJB33nkn4TTw8f79738zatQomjRpQtOmTbnsssv417/+BUC3bt0YMGAAoOniReo7tUDiHaGlkEwjRozgW9/6FgsXLqSgoIBBgwYxZcoU8vPzWbBgAenp6eTk5CScvv1o1q5dy4MPPshbb71Fq1atGDduXKTjlMjMzCxdTktLUxeWSD2mFkgN0LRpU4YOHcr1119fevN8165dtG/fnvT0dObOncsHH3xwxGOcddZZ/OlPfwJg2bJlLFmyBAimgW/SpAktWrRgy5YtzJz56YwvzZo1Y8+ePeWOdeaZZ/L8889TUFDA3r17mTZtGmeeeWZVna6I1BFqgdQQY8aMYdSoUaVdWWPHjuWSSy6hX79+xGIxevbsecT9b775Zq677jp69epFr169GDRoEACnnnoqAwcOpGfPnnTp0uWwaeDHjx/PBRdcQKdOnZg7d25peW5uLuPGjWPw4MEA3HDDDQwcOFDdVSJymBoxnXt10XTu1UfXVaTuqHHTuYuISO2mBCIiIpEogQD1qRuvOuh6itQP9T6BZGVlsX37dv2jV0Xcne3bt5OVlZXqUEQkyer9KKzs7GzWr19Pfn5+qkOpM7KyssjOzk51GCKSZPU+gaSnp9OtW7dUhyEiUuukpAvLzO4zsyVmtsjMXjazTgnqDDCzN8zsnbDulXHbppjZ2nD/RWY2oHrPQEREUnUP5AF37+/uA4AXgXsS1CkArnH3PsAFwCNmFj/X+O3uPiB8LaqGmEVEJE5KurDcfXfcahOg3B1sd38vbnmjmW0F2gF64pGISA2QslFYZjbRzD4CxpK4BRJfdzCQAayJK54Ydm09bGaZFewqIiJJkrQEYmZzzGxZgtcIAHe/2927AE8BXz/CcToCfwCuc/fisPhOoCfwWaA18N0j7D/ezPLMLE8jrUREqk7K58Iys67ADHfvm2Bbc+BV4EfunvDZrWZ2DnCbu198tPdKNBeWiIgcWY2aC8vMTolbHQGsTFAnA5gGPFk2eYStEix4PN9IYFnyohURkURS9TuQSWbWAygGPgBuAjCzGHCTu98AXAGcBbQxs3HhfuPCEVdPmVk7wIBFJfuLiEj1SXkXVnVSF5aIyLGrUV1YIiJS+ymBiIhIJEogIiISiRKIiIhEogQiIiKRKIGIiEgkSiAiIhKJEoiIiESiBCIiIpEogYiISCRKICIiEokSiIiIRKIEIiIikSiBiIhIJEogIiISiRKIiIhEogQiIiKRKIGIiEgkSiAiIhJJShKImd1nZkvMbJGZvWxmnSqoVxTWWWRmL8SVdzOz+Wa22sz+bGYZ1Re9iIhA6logD7h7f3cfALwI3FNBvX3uPiB8XRpX/hPgYXfvDuwAvpLkeEVEpIyUJBB33x232gTwyu5rZgacCzwbFv0eGFl10YmISGWk7B6ImU00s4+AsVTcAskyszwzm2dmJUmiDbDT3QvD9fVA5yO8z/jwGHn5+flVFr+ISH2XtARiZnPMbFmC1wgAd7/b3bsATwFfr+AwJ7p7DPgv4BEzO/lY43D3R9095u6xdu3aRT4fERE5XMNkHdjdz6tk1aeAGcCEBMfYEP5938xeBQYCzwEtzaxh2ArJBjZUSdAiIlJpqRqFdUrc6ghgZYI6rcwsM1xuC5wBLHd3B+YCo8Oq1wLTkxuxiIiUlap7IJPC7qwlwHDgvwHMLGZmj4d1egF5ZraYIGFMcvfl4bbvArea2WqCeyJPVG/4IiJiwRf6+iEWi3leXl6qwxARqVXMbEF4P/ow+iW6iIhEogQiIiKRKIGIiEgkSiAiIhKJEoiIiESiBCIiIpEogYiISCRKICIiEokSiIiIRKIEIiIikSiBiIhIJEogIiISiRKIiIhEogQiIiKRKIGIiEgkSiAiIhKJEoiIiESiBCIiIpEogYiISCQpSSBmdp+ZLTGzRWb2spl1SlBnaLi95LXfzEaG26aY2dq4bQOq/yxEROq3VLVAHnD3/u4+AHgRuKdsBXef6+4DwjrnAgXAy3FVbi/Z7u6LqidsEREpkZIE4u6741abAH6UXUYDM929IHlRiYjIsUjZPRAzm2hmHwFjSdACKeMqYGqZsolhN9jDZpZ5hPcZb2Z5ZpaXn59/nFGLiEgJcz/al/+IBzabA5yQYNPd7j49rt6dQJa7T6jgOB2BJUAndz8UV7YZyAAeBda4+w+PFlMsFvO8vLxjPhcRkfrMzBa4e6xsecNK7nwysN7dD5jZOUB/4El331nRPu5+XiVjewqYASRMIMAVwLSS5BEee1O4eMDMJgO3VfK9RESkilS2C+s5oMjMuhN84+8C/Cnqm5rZKXGrI4CVR6g+hjLdV2ELBDMzYCSwLGosIiISTaVaIECxuxea2SjgF+7+CzN7+zjed5KZ9QCKgQ+AmwDMLAbc5O43hOs5BMnqn2X2f8rM2gEGLCrZX0REqk9lE8ghMxsDXAtcEpalR31Td7+8gvI84Ia49XVA5wT1zo363iIiUjUq24V1HTAEmOjua82sG/CH5IUlIiI1XaVaIO6+HPgmgJm1Apq5+0+SGZiIiNRslWqBmNmrZtbczFoDC4HHzOyh5IYmIiI1WWW7sFqEvx6/jGD47mlAZYfpiohIHVTZBNIwHDp7BcHcVSIiUs9VNoH8EJhF8Ivvt8zsJGBV8sISEZGarrI30f8C/CVu/X0g4VBcERGpHyp7Ez3bzKaZ2dbw9ZyZZSc7OBERqbkq24U1GXgB6BS+/haWiYhIPVXZBNLO3Se7e2H4mgK0S2JcIiJSw1U2gWw3s6vNLC18XQ1sT2ZgIiJSs1U2gVxPMIR3M7CJ4AmB45IUk4iI1AKVSiDu/oG7X+ru7dy9vbuPRKOwRETqteN5pO2tVRaFiIjUOseTQKzKohARkVrneBJIch6mLiIitcIRf4luZntInCgMaJSUiEREpFY4YgJx92bVFYiIiNQux9OFVSXM7Ntm5mbWtoLt15rZqvB1bVz5IDNbamarzeznZqZ7MiIi1SilCcTMugDDgQ8r2N4amACcBgwGJoRPRAT4NXAjcEr4uiDpAYuISKlUt0AeBr5DxTfkzwdmu/vH7r4DmA1cED6bpLm7z3N3B54ERlZLxCIiAqQwgZjZCGCDuy8+QrXOwEdx6+vDss7hctlyERGpJpV6HkhUZjYHOCHBpruBuwi6r5LKzMYD4wG6du2a7LcTEak3kppA3D3hc9PNrB/QDVgc3vvOBhaa2WB33xxXdQNwTtx6NvBqWJ5dpnxDBTE8CjwKEIvF9NsVEZEqkpIuLHdfGs6plePuOQRdULllkgcEj9Edbmatwpvnw4FZ7r4J2G1mp4ejr64BplfnOYiI1HepvolejpnFzOxxAHf/GLgPeCt8/TAsA7gFeBxYDawBZqYgXBGResuCQUz1QywW87y8vFSHISJSq5jZAnePlS2vcS0QERGpHZRAREQkEiUQERGJRAlEREQiUQIREZFIlEBERCQSJRAREYlECURERCJRAhERkUiUQEREJBIlEBERiUQJREREIlECERGRSJRAREQkEiUQERGJRAlEREQiUQIREZFIlEBERCQSJRAREYkkpQnEzL5tZm5mbRNsG2Bmb5jZO2a2xMyujNs2xczWmtmi8DWgeiMXEZGGqXpjM+sCDAc+rKBKAXCNu68ys07AAjOb5e47w+23u/uz1RGriIiUl8oWyMPAdwBPtNHd33P3VeHyRmAr0K76whMRkSNJSQIxsxHABndfXMn6g4EMYE1c8cSwa+thM8s8wr7jzSzPzPLy8/OPL3ARESmVtARiZnPMbFmC1wjgLuCeSh6nI/AH4Dp3Lw6L7wR6Ap8FWgPfrWh/d3/U3WPuHmvXTg0YEZGqkrR7IO5+XqJyM+sHdAMWmxlANrDQzAa7++YydZsDLwF3u/u8uGNvChcPmNlk4LYknIKIiBxBtd9Ed/elQPuSdTNbB8TcfVt8PTPLAKYBT5a9WW5mHd19kwUZaCSwLOmBi4jIYWrU70DMLGZmj4erVwBnAeMSDNd9ysyWAkuBtsD9KQhXRKReM/eEg6DqpFgs5nl5eakOQ0SkVjGzBe4eK1teo1ogIiJSeyiBiIhIJEogIiISiRKIiIhEogQiIiKRKIGIiEgkSiAiIhKJEoiIiESiBCIiIpEogYiISCRKICIiEokSiIiIRKIEIiIikSiBiIhIJEogIiISiRKIiIhEogQiIiKRKIGIiEgkSiAiIhJJShOImX3bzNzM2lawvcjMFoWvF+LKu5nZfDNbbWZ/NrOM6otaREQghQnEzLoAw4EPj1Btn7sPCF+XxpX/BHjY3bsDO4CvJDFUERFJIJUtkIeB7wB+LDuZmQHnAs+GRb8HRlZtaCIicjQpSSBmNgLY4O6Lj1I1y8zyzGyemZUkiTbATncvDNfXA52P8F7jw2Pk5efnH3/wIiICQMNkHdjM5gAnJNh0N3AXQffV0Zzo7hvM7CTgH2a2FNh1LHG4+6PAowCxWOyYWjsiIlKxpCUQdz8vUbmZ9QO6AYuD3iiygYVmNtjdN5c5xobw7/tm9iowEHgOaGlmDcNWSDawIVnnISIiiVV7F5a7L3X39u6e4+45BF1QuWWTh5m1MrPMcLktcAaw3N0dmAuMDqteC0yvthMQERGghv0OxMxiZvZ4uNoLyDOzxQQJY5K7Lw+3fRe41cxWE9wTeaL6oxURqd+S1oVVWWErpGQ5D7ghXP4P0K+Cfd4HBldHfCIikliNaoGIiEjtoQQiInVCcbFTXHzkgZbuzv5DRQm37Sw4SFGZ/acv2kDOHS8xaebKKouzLkl5F5aI1A/ujjs0aGDHtF9xsbOj4CCtm2TgDkXupKcF330/+riA/6zZxnefW1pa/827h9E8K52MtAZs3LWPz/9kLtcOOZHenZqX1vtMh6Y8d/PnKDhYxMIPdpDRsAFf+X1ehTH85p9rKHbnrot6RTjzqlFYVMzKzXvo27lFuW2fHCikSUYa4cjWUjsLDrJuewEZaQ3o3al5lcdkwaCm+iEWi3leXsUfEhFJjr0HCukzYRYAF/Y9gZnLNjPlus/SOKMhV/z2DQAaGPzz9qF0ad2YNfmf8MqKLazdVsDUN48021Hynf2ZdvzzveBHyH//nzP596pt3P/SCjq3bMSGnfsAuOPCnqWtlJe/dRYtG6XTvnkW7s6qrZ+Q06YJGQ2DpLd66x7Oe+i10uPfN7IvXz79RPYfKmLe+9spLHJaNk5n0ImtWJO/l2fyPmLsaV05+4FXAWjRKJ2fXN6Pc3t2YEfBQf66cAM/+Xvw3j1PaMZj18To0roxf1u8kW9Mfbv0fabd8jkGdm0V6RqY2QJ3j5UrVwIRqV1eX72Nftkt+PITbzI6tzNdWjfmjO5tS7+VVxV3Z9mG3RwsKmLQia0pLCqmYVoDtu7Zz1trd/C1Py0E4P0fXcRPZ73Lb/65hjsv7Mmkv6/ksoHZ3DuiDzf/cQH/WrWtSuNK5Izubbi4fyfu/OvSI9a7+vSu9OrYnLunLSu37ceX9aNd00ze3/YJP5qxkrM+0459Bwt55qtD+OP8D/n+8+X3ORbNsxryj9vO4bQfvVKuq2xYz/a8snLrcR3/SNIaGKsnXliuhVJZSiAogUjtsmvfIYqKncYZaQC8/eFO7pq2lLXb9h5xv5JvtBB08bRqksGLizfyhd4daNM0s1z9goOFGIYZ7N53iIUf7uCmPy6s+hMCpt54OmMem3dYWaP0NG48sxubd+/nmbz15fYZe1pX7rqoF30mzGLiqL6szd/LoaJiHLj9/B40y0pP+F6rt+7ha0+9zZTrP0vHFo2OO/ZXVmwp183VvX1TbjyzG/f+bTlfHnIiv/3n+5U+3u3n9+CBWe9Wuv7ludlMuLQ3/X/wcrltY0/ryreH9yD3vtnlyieOSjiY9ZgogaAEItXrD/M+OOxb69vf/wKtmmSw/1ARWelBUti0ax9jH59PbtdWTLikNy+/s4VTu7Tg78s28+DL7yUlrs4tg39MS7pfohraox1z3z36/HI/u2oAl57aqVLffncVHOKh2e8y9vQT+UyHZscVXzK4O6u3fsJJ7ZqSdpR7OY+99j6tm2Rw+aBsdu07xKn3fvoP/6u3nUNO2yZAcP+i74RZDOjSkue/dkZpna2799MsK51G4ReIREpahWVj3FlwiJaN0yO3OMpSAkEJRI5fcbFTWOxkNGyAu3PGpH+wcdf+Yz5Oh+aZbNl9IFIMYwZ3oYEZX+zXkQFdW5LZMI1DRcVkpafx6rtbGTf5rUjHjdc/uwUPjD6VHic0Y/+hIu6atpR7Lu7Ntk8O0K1tU/YfKqJJZkOKi53pizdwWrc2dGp5/N/ypWZSAkEJpL746OMCnpr/Ie2bZXL957sBcLCwmCn/Wcup2S05oUUWXVs3Lh0R9OOZK8hqmMbu/YeY/Po6vtC7A18f2p29BwoZcnIbxjw2j3nvf0x2q0bs2V/Irn2HKh3Lxf07sqPgIK+v3n7M5zGsZ3ueGPfZw8oOFRUf9V7HgcIiNuzYR8vGGbRslM6qrZ/QuVUjmmY25EBhEd95dgmxnNYMOak1ew8UkdbA6Nu5BSX/FlTVt1apO5RAUAKpS9ydGUs387U/LaRpZkMevzbG6Se1AaDX9//OvgrG+ifDnFvPYsvuA5zRvS0LPthBjxOaMWPJJr7z3BIWfv8LtG6SwfZPDnD+I6/xrS98hgv6nMCg++fQr3MLnvnqkCN2UYjUBEogKIFUtUNFxcx7fzu/fnUN/1mznRaN0pl961m0aZLJJ/sLadE4vfSHXQ++/C6vrcpnbf5efnBpH/7v1TVAMDJl8fpdZKQ1YNGEL3DD7/P4z5pPv63//vrBnP2ZduTvOUCbJhnsKDjIoPvnVPu5tm2aybZPgi6nV287hyn/WcctQ0+mfbOsao9FpLopgVD1CeStdR/ToVkWXds0BmD3/kOs/3jfMf9gZ03+J8x7fztjTzvxmGNwd4qKnYZpDfhwewGZ6Q3o0DyLjz4u4K8LN/Delj28tHQTv/yvgXRv35S3P9zJlbEuNGhgFBc7SzfsYsSvXi893l9v+RwDslsCQfeOuzN7+RZmvbOF5xZ+OkJmxIBOTF+08ZjjrWqXntqJFxaXj+Onl/dnWK/2DLp/Do0z0ph/17DS0To7Cw7y7IL1DDm5DR2aZ9G2aSY79h6kaVbDKh8KK1IXKIEQPYFs3rWf6Ys2cMOZJzH59bXc/9KKcnXuvqgXE2eUL//xZf24uH9HFn64k/FP5nGgsJhvnNudrPS0ckP4Bp3Yir98dQjbPjlA++aJv9nuP1TEyF+9zi//ayAHC52Lfv6vYz4fgFOzW7B4/ZGfzfX9i3tz34vLj1gnWTo0z+SaITkVDnOcf9cwOsRdo4KDhfS+J/ih2nVn5DDhkj7VEqdIfaAEQvQE0m/CLPYcKDx6xST4bE4rNu4MRvkc77DL45XTpjG3DO3Ov1dt44XFG/nGud359vAe5eo9+cY6hvXqQGbDBrRtmsk/Vm6hb+cWR+zuOVhYzJbd+8lu1eiwm7gHC4vZvf8QbZpkMG7yW/Tr3ILbzi//niKSPEogRE8gOXe8VK5s4qi+XJ6bTXpaA6787RvkfbADCL4Zf3KgkGH/+8+Ex2qa2ZBPwmRUMhVCh+aZPPilU/nyE28ee2xtGjNqYDYPz3mPt7//BfbsL2TF5t2c3K4p3ds3La1XXOxs2LmPLq0b842pb/O3xRv54Yg+XDMk57DjuTt7Dxbxt8UbS3/Vu27SF9l3sEg3e0XqKSUQqiaBPPrlQQzvU/5R7+5OsVPux0W/eGUV/zv7PX5wSW++FOtCk8yK56+csXQTk2au5NmbhrB+5z4u+7//HLb9e1/sxeZd+7lmSA4dWmSyq+BQhV1dVWHK62sZ3K1NUiZhE5HaQwmE6Anke88v5Y/zggnd3rn3/CMmgUSKi/2YZyAVEakpKkogms69EoqKnfbNMnnz7vMi7a/kISJ1UUrHLJrZt83Mzaxtgm1DzWxR3Gu/mY0Mt00xs7Vx2wYkM84Dh4rJTNfwThGReClrgZhZF2A4kHCyf3efCwwI67YGVgPx01De7u7PJjtOgL0HC8lsqBvIIiLxUvm1+mHgO0BlbsKMBma6e0FyQ0rs1C4tOa9Xh1S8tYhIjZWSFoiZjQA2uPviSk7cdhXwUJmyiWZ2D/AKcIe7J5za1MzGA+MBunbtGineW87pHmk/EZG6LGmjsMxsDlB+vCvcDdwFDHf3XWa2Doi5e8LHlplZR2AJ0MndD8WVbQYygEeBNe7+w6PFpLmwRESOXbWPwnL3hEOWzKwf0A0oaX1kAwvNbLC7b06wyxXAtJLkER57U7h4wMwmA7dVafAiInJU1d6F5e5LgfYl60drgQBjgDvjC8yso7tvsiADjQSO72HFIiJyzGrU2FQzi5nZ43HrOUAXoOy8IE+Z2VJgKdAWuL+6YhQRkUDKf0jo7jlxy3nADXHr64DOCfY5tzpiExGRitWoFoiIiNQeSiAiIhKJEoiIiERSr2bjNbN84IOIu7cFKhopVl/pmpSna1Kerkl5te2anOju7coW1qsEcjzMLC/RD2nqM12T8nRNyj+X4iYAAAg8SURBVNM1Ka+uXBN1YYmISCRKICIiEokSSOU9muoAaiBdk/J0TcrTNSmvTlwT3QMREZFI1AIREZFIlEBERCQSJZBKMLMLzOxdM1ttZnekOp5kMbMuZjbXzJab2Ttm9t9heWszm21mq8K/rcJyM7Ofh9dliZnlxh3r2rD+KjO7NlXnVFXMLM3M3jazF8P1bmY2Pzz3P5tZRlieGa6vDrfnxB3jzrD8XTM7PzVnUnXMrKWZPWtmK81shZkNqe+fFTP7Vvj/zjIzm2pmWXX6s+Lueh3hBaQBa4CTCB5gtRjoneq4knSuHYHccLkZ8B7QG/gpwVMfAe4AfhIuXwTMBAw4HZgflrcG3g//tgqXW6X6/I7z2twK/Al4MVx/BrgqXP4NcHO4fAvwm3D5KuDP4XLv8LOTSfA8nDVAWqrP6zivye+BG8LlDKBlff6sEEz8uhZoFPcZGVeXPytqgRzdYGC1u7/v7geBp4ERKY4pKdx9k7svDJf3ACsI/qcYQfCPBeHfkeHyCOBJD8wDWoZPizwfmO3uH7v7DmA2cEE1nkqVMrNs4IvA4+G6AecCz4ZVyl6Tkmv1LDAsrD8CeNrdD7j7WmA1wWerVjKzFsBZwBMA7n7Q3XdSzz8rBDOcNzKzhkBjYBN1+LOiBHJ0nYGP4tbXk2CK+bombE4PBOYDHfzTp0BuBjqEyxVdm7p2zR4BvgMUh+ttgJ3uXhiux59f6bmH23eF9evaNekG5AOTw669x82sCfX4s+LuG4AHgQ8JEscuYAF1+LOiBCLlmFlT4Dngf9x9d/w2D9rY9Wbst5ldDGx19wWpjqWGaQjkAr9294HAXoIuq1L18LPSiqD10A3oBDShdremjkoJ5Og2EDwVsUR2WFYnmVk6QfJ4yt3/GhZvCbsbCP9uDcsrujZ16ZqdAVwaPnr5aYLuiJ8RdMGUPJAt/vxKzz3c3gLYTt26JhB8K17v7vPD9WcJEkp9/qycB6x193x3PwT8leDzU2c/K0ogR/cWcEo4kiKD4GbXCymOKSnC/tcngBXu/lDcpheAktEx1wLT48qvCUfYnA7sCrsvZgHDzaxV+K1seFhW67j7ne6e7cGTM68C/uHuY4G5wOiwWtlrUnKtRof1PSy/Khx50w04BXizmk6jyrn7ZuAjM+sRFg0DllOPPysEXVenm1nj8P+lkmtSdz8rqb6LXxteBCNI3iMYDXF3quNJ4nl+nqDLYQmwKHxdRNAv+wqwCpgDtA7rG/Cr8LosBWJxx7qe4ObfauC6VJ9bFV2fc/h0FNZJBP9Trwb+AmSG5Vnh+upw+0lx+98dXqt3gQtTfT5VcD0GAHnh5+V5glFU9fqzAtwLrASWAX8gGElVZz8rmspEREQiUReWiIhEogQiIiKRKIGIiEgkSiAiIhKJEoiIiESiBCJ1ipkVmdkiM1tsZgvN7HNHqd/SzG6pxHFfNbNY1UVa+5nZFDMbffSaUlcpgUhds8/dB7j7qcCdwI+PUr8lwayoNVLcL5hFahwlEKnLmgM7IJjfy8xeCVslS82sZEblScDJYavlgbDud8M6i81sUtzxvmRmb5rZe2Z2Zlg3zcweMLO3wudcfDUs72hmr4XHXVZSP56ZrTOzn4bv9aaZdQ/Lp5jZb8xsPvBTC56x8Xx4/Hlm1j/unCaH+y8xs8vD8uFm9kZ4rn8J5zbDzCZZ8KyXJWb2YFj2pTC+xWb22lHOyczslxY8o2IO0L4q/2NJ7aNvN1LXNDKzRQS/8u1IMHcVwH5glLvvNrO2wDwze4FgAsC+7j4AwMwuJJgQ7zR3LzCz1nHHbujug83sImACwdxHXyGYluOzZpYJvG5mLwOXAbPcfaKZpRFM7Z3ILnfvZ2bXEMz6e3FYng18zt2LzOwXwNvuPtLMzgWeJPgV+PdL9g9jbxWe2/eA89x9r5l9F7jVzH4FjAJ6urubWcvwfe4Bznf3DXFlFZ3TQKAHwfMqOhBM0/G7Sv1XkTpJCUTqmn1xyWAI8KSZ9SWYSuNHZnYWwbTsnfl0qvF45wGT3b0AwN0/jttWMrnkAiAnXB4O9I+7F9CCYO6it4DfWTA55fPuvqiCeKfG/X04rvwv7l4ULn8euDyM5x9m1sbMmoexXlWyg7vvsGD24N4E/+hD8KCnNwimCt8PPGHBUxVfDHd7HZhiZs/EnV9F53QWMDWMa6OZ/aOCc5J6QglE6ix3fyP8Rt6OYE6vdsAgdz9kwey6Wcd4yAPh3yI+/X/HgG+4e7kJAMNk9UWCf6AfcvcnE4VZwfLeY4yt9G0JHtA0JkE8gwkm+BsNfB04191vMrPTwjgXmNmgis4pbHmJlNI9EKmzzKwnwSOJtxN8i94aJo+hwIlhtT0Ej+8tMRu4zswah8eI78JKZBZwc9jSwMw+Y2ZNzOxEYIu7P0bwJMPcCva/Mu7vGxXU+RcwNjz+OcA2D57TMhv4Wtz5tgLmAWfE3U9pEsbUFGjh7jOAbwGnhttPdvf57n4PwQOiulR0TsBrwJXhPZKOwNCjXBup49QCkbqm5B4IBN+krw3vIzwF/M3MlhLMILsSwN23m9nrZrYMmOnut5vZACDPzA4CM4C7jvB+jxN0Zy20oM8on+CRpecAt5vZIeAT4JoK9m9lZksIWjflWg2hHxB0hy0BCvh0CvD7gV+FsRcB97r7X81sHDA1vH8BwT2RPcB0M8sKr8ut4bYHzOyUsOwVgmdxL6ngnKYR3FNaTjB1eUUJT+oJzcYrkiJhN1rM3belOhaRKNSFJSIikagFIiIikagFIiIikSiBiIhIJEogIiISiRKIiIhEogQiIiKR/D+k4ZUFWWIpLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-07-16T14:30:45.729Z"
    }
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T07:40:05.307438Z",
     "start_time": "2019-07-26T07:00:15.422902Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>👉🏻LMAE👈🏻</th>\n",
       "      <th>lmae0</th>\n",
       "      <th>lmae1</th>\n",
       "      <th>lmae2</th>\n",
       "      <th>lmae3</th>\n",
       "      <th>lmae4</th>\n",
       "      <th>lmae5</th>\n",
       "      <th>lmae6</th>\n",
       "      <th>lmae7</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-2.846424</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-2.855474</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-2.861769</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-2.867633</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>-2.868024</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>-2.887645</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>-2.902774</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>-2.912449</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>-2.919319</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>-2.933897</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>-2.944091</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>-2.955310</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>-2.962945</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>-2.968143</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>-2.974393</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>-2.979317</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>-2.987009</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>-2.989137</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>-2.985991</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>-2.990614</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(20, 1e-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optional Fine tune regular fit "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T12:03:48.800934Z",
     "start_time": "2019-07-18T12:03:48.795708Z"
    }
   },
   "outputs": [],
   "source": [
    "learner.callbacks.append(\n",
    "    ReduceLROnPlateauCallback(learner, monitor='train_loss', mode='min', factor=0.2, patience=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:55:46.972894Z",
     "start_time": "2019-07-26T06:49:05.682183Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='3' class='' max='100', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      3.00% [3/100 05:38<3:02:29]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>👉🏻LMAE👈🏻</th>\n",
       "      <th>lmae0</th>\n",
       "      <th>lmae1</th>\n",
       "      <th>lmae2</th>\n",
       "      <th>lmae3</th>\n",
       "      <th>lmae4</th>\n",
       "      <th>lmae5</th>\n",
       "      <th>lmae6</th>\n",
       "      <th>lmae7</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>-2.742189</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-2.745732</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>-2.752410</td>\n",
       "      <td>#na#</td>\n",
       "      <td>01:52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='136', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-86-acb2e43d0021>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m# , moms=(0.75,0.70))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mto_detach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-15-a8bc1c69b8fd>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_outputs, t_scalar, t_magnetic, t_dipole, t_potential)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_types\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexclude_ext\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mext\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m                 \u001b[0m_output\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_target\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp_scalar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt_scalar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# scalars at the end\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                 \u001b[0;31m# LMAE scalar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learner.fit(100, 1e-3)# , moms=(0.75,0.70))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.fit_one_cycle(300, 5e-4)#, moms=(0.75,0.70))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_lr(show_moms=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T08:58:42.510560Z",
     "start_time": "2019-07-26T08:58:42.225753Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEGCAYAAABLgMOSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3xV9f348dc7m0wIe4ehshGILEVEqKKiuGq1ah1V1NpqbauFuqgTR6219lt/1FWtotaFiijDiRUQkA0CyhDCHgmQnbx/f5xzb+7NvTe5CUluxvv5eOSRe8/5nJPPISHvfNb7I6qKMcYYU1VRka6AMcaYhskCiDHGmGqxAGKMMaZaLIAYY4ypFgsgxhhjqiUm0hWoS61atdKMjIxIV8MYYxqUpUuX7lPV1uWPN6kAkpGRwZIlSyJdDWOMaVBEZGuw4xHrwhKRx0RkvYisFJF3RKR5iHK3icgaEVktIjNEJKHc+adE5Ejd1NoYY4xHJMdA5gL9VHUAsAGYUr6AiHQEbgEyVbUfEA1c6nM+E2hRN9U1xhjjK2IBRFXnqGqx+3Yh0ClE0RigmYjEAIlAFoCIRAOPAXfUdl2NMcYEqi9jINcCr5c/qKo7RORxYBuQB8xR1Tnu6V8D76nqThEJeWMRmQRMAujSpUtN19sYEwFFRUVs376d/Pz8SFelUUlISKBTp07ExsaGVb5WA4iIzAPaBTl1p6rOdMvcCRQDrwS5vgUwEegGHAL+KyJXAJ8APwVOq6wOqjodmA6QmZlpib+MaQS2b99OSkoKGRkZVPQHpAmfqrJ//362b99Ot27dwrqmVgOIqo6r6LyIXA1MAMZq8KyO44DNqrrXLf82MBI4CPQENrk/PIkisklVe9Zg9Y0x9VR+fr4FjxomIrRs2ZK9e/eGfU3EurBEZDzO+MVoVc0NUWwbMFxEEnG6sMYCS1R1Fj4tGxE5YsHDmKbFgkfNq+q/aSRnYT0NpABzRWS5iDwDICIdRORDAFVdBLwJLANW4dR3eoTqWyWrd2SzdOvBSFfDGGNqTcRaIKFaDKqaBZzt8/5e4N5K7pVcs7U7dhP+vgCA8wZ2YMKA9pzRN9hQkDGmIdq/fz9jx44FYNeuXURHR9O6tbNQe/HixcTFxVV6j2uuuYbJkydzwgkn1Gpda1N9mYXVaL23Iov3VmSxZdo5ka6KMaaGtGzZkuXLlwMwdepUkpOT+cMf/uBXRlVRVaKignf0vPDCC7Vez9pmyRTrSGmp8uOBUEM9xpjGYNOmTfTp04fLL7+cvn37snPnTiZNmkRmZiZ9+/blvvvu85Y95ZRTWL58OcXFxTRv3pzJkyczcOBARowYwZ49eyL4FOGzFkgtOFJQHHDsyfkbeWr+Rj767Sh6tUut0v3eW5HF0Ix02qUlVF7YmCbmz++vYW1WTo3es0+HVO49t2+1rl2/fj0vvfQSmZmZAEybNo309HSKi4sZM2YMF198MX369PG7Jjs7m9GjRzNt2jR+97vf8fzzzzN58uRjfo7aZi2QWnAkPzCAvLnkRwC2H8ir0r3yi0q4Zca3/PxfC2ukbsaY2tWjRw9v8ACYMWMGgwcPZvDgwaxbt461a9cGXNOsWTPOOussAIYMGcKWLVvqqrrHxFogteBoYWAAycp2VsyuycphXJ+2AeePFBSTHB/47cgtLAHgh31Ha7iWxjQO1W0p1JakpCTv640bN/K3v/2NxYsX07x5c6644oqgq+d9B92jo6MpLg78HVIfWQukFuS5v/SD+eu8DfS/92Pyi5wye3LyyZg8i373fkzG5Fnk5BcBUFBcQn5RCV9t2lcndTbG1LycnBxSUlJITU1l586dfPzxx5GuUo2yFkgtOBpkDMTX4YJith/Mo2ebZB79+Du/c68u2saNo3tw+uOfs+NQWXdX88TwctMYY+qPwYMH06dPH3r16kXXrl05+eSTI12lGmUBpBbkFoVugXis35VDzzbJnNi5OW8u3e49Pm32ei4c1NEveAB0SU+s8XoaY47d1KlTva979uzpnd4Lzsrul19+Oeh1CxYs8L4+dOiQ9/Wll17KpZdeGuySese6sGpBRV1YHr9+9VsmvbTE25Xl69ynFwQcW7k9m4/X7CJj8iw+Wr2zRuppjDHHwgJIDVublcOiH/b7HUtPCr4qdc7a3by/MjAY7M4pCFr+hpeXAnDjf5Z5x0qMMSZSLIAcg7zCEn7/xgr2HXF+4a/ans3ZT33Jv78u2z74rnN6M/93o5l726ncf36/gHvkuuMlGS2r1kVV0/PejTGmqiyAHIP3VuzgrWXbOXnaJ0Bg19PdE/pwzcndaJEUx3FtU7hyeFe2TDuHf14+2Ftm454jtEiM5c2bRlbpa5eW2tYmxpjIsgByDKLdHDcFxaVBz18zMoPoqMD0yOP7teOFa07yvj9aWEKr5Hhm3XJK0PvcOva4gGM/f3YRGZNnsSfHdmQzxkSGBZBjEBvtHxx6tUvxex8VJHiAMzNjzAltvO8L3QDUt0Ma7/26bJrfaSe0plVyPBcP6cTATmlB7zX0ofkV1nH/kQJ2Zldt9bsxxoTDAsgxiIsu++f7cuNe4mOjve9vG3d8te45oFNzurdK8r5ectc4OqcnMrxHy2rdb8gD8xjx8CfVutYYE9yYMWMCFgU++eST3HTTTSGvSU52dp3Iysri4osvDlrmtNNOY8mSJRV+7SeffJLc3LLErGeffbbfNOC6ZAHkGBT5jEN8v+eId1wiJSGGW8cFdjuFy5O2JMtnLUhCTFlwuum0Hn7ll/8Y/Idn4+7D3td7Dwef2WWMqbrLLruM1157ze/Ya6+9xmWXXVbptR06dODNN9+s9tcuH0A+/PBDmjdvXu37HYuIBBAReUxE1ovIShF5R0SCPr2I3CYia0RktYjMEJEE97iIyIMiskFE1onILXVZ/ylvr+LNpdsp8FnDUap4xzse/+nAY7r/HeOdDWZ8e8DOHdgBgDduGMGNo/0DyCOz1wNO4kXfVfAzl2d5X++2sRJjaszFF1/MrFmzKCwsBGDLli1kZWUxaNAgxo4dy+DBg+nfvz8zZ84MuHbLli306+fMyMzLy+PSSy+ld+/eXHDBBeTllf3ReNNNN3nTwN97r7On3lNPPUVWVhZjxoxhzJgxAGRkZLBvn5Py6IknnqBfv37069ePJ5980vv1evfuzfXXX0/fvn0544wz/L7OsYjUSvS5wBRVLRaRR4ApwB99C4hIR+AWoI+q5onIG8ClwIvA1UBnoJeqlopIG+qIqjJj8TZmLN7GAz7TchPjotmZncdFgztxZpi7D0aJE3imXznE7/j1o7pzJL+Ya0/p5j3Ws01yyE2pdh/O55stB5j81kq27M/l+4fO5rTHPmXL/rK/UvYczgeCj6MY06DNngy7VtXsPdv1h7OmhTydnp7O0KFDmT17NhMnTuS1117jkksuoVmzZrzzzjukpqayb98+hg8fznnnnRdyr/F//vOfJCYmsm7dOlauXMngwWUzNB988EHS09MpKSlh7NixrFy5kltuuYUnnniCTz/9lFatWvnda+nSpbzwwgssWrQIVWXYsGGMHj2aFi1asHHjRmbMmMG//vUvLrnkEt566y2uuOKKY/5nikgLRFXnqKrnT+WFQKcQRWOAZiISAyQCnj+pbwLuU9VS9351tvvK+l1l3UK+s68mv72K3TkFtK/Cnh0f3jqKP5/XN2C729joKO4Y34tWyfEhrx3RvSVtU53zPVsn89Nnvub7vUcpKVVyC4v9ggfArmzrwjKmJvl2Y3m6r1SVP/3pTwwYMIBx48axY8cOdu/eHfIeX3zxhfcX+YABAxgwYID33BtvvMHgwYMZNGgQa9asCZoG3teCBQu44IILSEpKIjk5mQsvvJAvv/wSgG7dunHiiScCNZsuvj7kwroWeL38QVXdISKPA9uAPGCOqs5xT/cAfiYiFwB7gVtUdWOwm4vIJGASQJcuXY6poiWlyjS3uwicjLnlHcorDPt+vdqlVnlzKY8Zk4YDMP7JLyi/JKTPPYEZP3dZF5ZprCpoKdSmiRMnctttt7Fs2TJyc3MZMmQIL774Inv37mXp0qXExsaSkZERNH17ZTZv3szjjz/ON998Q4sWLbj66qurdR+P+PiyP0ajo6NrrAur1logIjLPHbso/zHRp8ydQDHwSpDrWwATgW5AByBJRDxtrnggX1UzgX8Bz4eqh6pOV9VMVc30bHpfXU/O28DnG/Z63xcUBa7/8B3srgvJ8TEcOFpx66JVchx7D1sAMaYmJScnM2bMGK699lrv4Hl2djZt2rQhNjaWTz/9lK1bt1Z4j1NPPZVXX30VgNWrV7Ny5UrASQOflJREWloau3fvZvbs2d5rUlJSOHz4cMC9Ro0axbvvvktubi5Hjx7lnXfeYdSoUTX1uEHVWgtEVcdVdF5ErgYmAGNVNdiy6nHAZlXd65Z/GxgJ/AfYDrztlnsHqJPd6V//5ke/9wXFpcTHRPl1ZXWtYkqSY5WcEFNpWpPUZrHkBNkl0eONb35k6daDPHLxgJBljDGBLrvsMi644AJvV9bll1/OueeeS//+/cnMzKRXr14VXn/TTTdxzTXX0Lt3b3r37s2QIc546MCBAxk0aBC9evWic+fOfmngJ02axPjx4+nQoQOffvqp9/jgwYO5+uqrGTp0KADXXXcdgwYNqtXdDSX47+7aJSLjgSeA0Z4AEaTMMJyWxUk4XVgvAktU9e8iMg3YoKrPi8hpwGOqelKw+/jKzMzUyuZYV+T8f3wVMGU2NSHG75fz9w+dHXT1eW25+dVlzAqSkNHXwE5ppCXG8dK1QwPOrcnK5pynnBQsS+4aV+G4izH1xbp16+jdu3ekq9EoBfu3FZGlbo+Pn0itA3kaSAHmishyEXkGQEQ6iMiHAKq6CHgTWAascus63b1+GnCRiKwCHgauq4tK/3xo4BhKfGw0HXwGzusyeEDZKvbyVk09g9vPPIH7J/YlO6+ILzYExmlV9QYPgMwH5vH2su0B5YwxJpiIDKKras8Qx7OAs33e3wvcG6TcISD4nNZasONQHodyCyFIbIiPieLNG0cy/OGKU4rUlrlr/Wd4fHnHGPYcLiAlIZabxzj/zHfPXAM4AcMznXDu2t2s3pEdcL/fvbGCXu1SOVJQzL4jBZzdv30tP4ExpqGqD7Ow6r1/fraJD1ft4o4zTwg4Fx8TRbu0BBZOGYtS992BvzylG88t2AzAuzefTOf0RDqX273wtnHH89d5GzhSUExKgrM17vUvhe7KO/upL72vV//5TJLjq/Zj8sY3P9IsLtq7+NGY2uD7B5GpGVUd0rBUJmEQBFUNmC4LcMRd+d0uLYH2ac3quGbwp7PL+iqT4oLPAGvjrhc5Usle7cEUhegiq8gdb63kNzO+rfIPozHhSkhIYP/+/fYzVoNUlf3795OQEP5aNmuBhEEEFCgN8sMaavfAuuI75pIYoqWQ5B4/WlDMy19vCZklOJic/CJahNhRMZjtB8sWMO7OKaBdFRZWGhOuTp06sX37dvbuDToHx1RTQkICnTqFWtcdyAJIGKJEKC1VvwCSHB9Trb/oa1OoFkiKG0D2HSn0jofERUdRWOK0LjY+eBb7jhSwJ6eAif/4yu/al77eyt0T+oRdh8M+M9JGTJvP5ofrbKjKNCGxsbF069at8oKmVlkXVpgU/10AP/3DaQDcEmSzp0hJjAv+90AzN7Dc9e5q7zFP8AAndUr7tGYM7Nw8YNziSAXrR4J559sd3tfWu2BM42YBJAwizl/WH63Z5T3WOiWeLdPO4Xc/qd6+HzXp0YsHMKBTGnExwb+dg7o4yY437TkScO72chMD8t0MwzFRQv+OaeysYgqU6V/84Pf+NzO+rdL1xpiGwwJIGMSdv7vwhwMRrklwl2R25r1fB98OFyA+Jpph3dIDjovgnerrcVY/J7FjlAhtUxPYnX1sKVDeX5FVeSFjTINkASQMdbw2sFaUn9oL8Nr1wwOOnTuwA73apXD3uX1olxbP7irk0PKdEeO7uNJmyhjTOFkACUP5qea/PYbdBiOlS5AA0qt9YCbg2OgoPvrtqVw5vCvtUhM4lFvk7daqzJ0+Yyy++cEKqjEV2BhT/1kACUP5xUq/HhN0IX29Nq53W+/rr6eczvcPnU1as9gKr2mb6rQiwt3N8NVF27yvX/7lMO/rYGMvxpiGzwJIGMr3YNV1vqua0Cm9bJFjcnxMWM/gWcMxe/WukGU+Xb+HnPwiv2PXntyNPh1S+Yu7te+Evy8IdqkxpoGzABKG8i2Qhpg+ITWhrLURbmoSTznfTbR8Hcot5JoXv2HA1Dl+4xwZrZzuslYpltnXmMbMAkgYGmC8CCrdXVEebgCsrJWS5zM24snHBXAo12mRJISYVmyMaRxsJXoYGkn84JPfjyY/yC6KoRSVlLUqCotLA9aZ5BWWBZAHZq3zvr5gUEcABndt4T1WWqp+KVRW78imT/vUKqVVMcbUL/YnYhgaSwukeWJclXJTtfHpgvrrvA0B5/OCzM76/PbTvFOGY6OjuH9iXwCWbz9Esbv6ffoX3zPh7wt4c6ntPWJMQxaxACIij4nIehFZKSLviEjzEOVuE5E17n7qM0QkwT0+VkSWuRtSLRCRWpsaFeUTQUb2aFlbX6be6Zye6B0HWb7tUMD5YNN7u7ZM8nt/QjtnqvCF//c/et7p7Ov83S5nVlZ9yyVmjKmaSLZA5gL9VHUAsAGYUr6AiHQEbgEyVbUfEA1c6p7+J3C5qp4IvArcVVsVbSQNkGpZNfUMAAZ0Tgs4l1voH0BSEgJ7RNuUG0h30uI7XWO++biMMQ1PxAKIqs5RVc+foAuBUDmEY4BmIhIDJAKe3BgKeFbCpfkcr3k+LZCmtqhaRGiZFMfRIK2FvHIB5Nu7fxJQxrMXicf2g3nk5DmD7Nl5RQHljTENR30ZRL8WeL38QVXdISKPA9uAPGCOqs5xT18HfCgieUAOEJiXAxCRScAkgC5dAvc0D4dvCyQSuw5GWlJ8TNCsvJ4xkI9/eyqdWjQjJjrw75HyGYLv/2Atew47e6hYADGmYavVFoiIzHPHLsp/TPQpcydQDLwS5PoWwESgG9ABSBKRK9zTtwFnq2on4AXgiWB1UNXpqpqpqpmtW7eu1nP4joEMzQhMStjYhdr7xNMCSUmI8W5aVZmSUvWubM/OtQBiTENWqy0QVR1X0XkRuRqYAIzV4Bn3xgGbVXWvW/5tYKSIfAwMVNVFbrnXgY9qrOIB9XQ+j+3VhlvHRT59e11Ljo8hJ6+YC//vKy4c3IkrhndFVfnq+/0ApFaSEsVXZkY689fvAawFYkxDF7EuLBEZD9wBjFbV3BDFtgHDRSQRpwtrLLAEOAikicjxqroB+AmwLsQ9jr2u7ufurZMaZBqTY5WcEMOybQc5lFvEsm2HGNYtnYWbD3hTtYfaCTGYRz4qW9W+YNM+Fv6wn+Hdm87MNmMak0jOwnoaSAHmulNxnwEQkQ4i8iGA28J4E1gGrMKp73R38P164C0RWQFcCdxeWxX1tEAaYgqTmpAUH+NdXQ5w3wdrWflj2bTeyv5dPBtaxQQJvi98tTngmDGmYYhYC0RVg67bUNUs4Gyf9/cC9wYp9w7wTq1V0IfnF2TTDB+wZke23/u9hwu8aVHC8dqk4ew9XMApj3zqPRYTJRSXqqV6N6YBs5XoYWiiDQ+vO8b7b3u7ftfhsAfNwdkRsVML//1Iit395QstgBjTYFkAqYomGkjG9GrjfX1mX2dfEd+9P6rqjRtG8Mp1zn4hnpZMXmEJxSWlfLJ+Nze+vNQvu29hcantamhMPVRf1oHUa57fXdJEI0icu75jfN92tE099hTtg7o0JzY6iv4d01ix3RlL6X2P/yS6lxdu5RcjMth3pIDMB+Zx77l9uObkbsf8tY0xNcdaIFXQVLuyRIQt087hmSuHEFtuseDXU04P+z6vTRrOdad0895j1Y5sfjyQx74jBQFl75m5BoCsQ3kAvPi/LdWsvTGmtlgLJAye7pMmGj/8tEwua4F0b51E+7RmFZT2N7x7S78pu5cN7cyMxT+S+cC8oOXP/fsCb3DZuj83aEp5Y0zk2P/GMHi7sCyC8MtTyrqRdhzMO6Z7XZLZOeS5tGaxrNqRzc7ssv3Y9xwOb292Y0zdsAASBs/wbVMdA/EVFxPFNSdnABzzFNwTOwfN4E9m1xYUFAemij9kqU+MqVcsgFSBtUAcvdunVl4oDMEWIA7p2oLh3VsG3TnxYG5hjXxdY0zNsAASBptB6q9dqrOr4XFtko/5XqOPL0twec3JGUy/cgipzYIPzR20Fogx9YoNoofBk8K9qaYyKa9TC2fg3JNV91hMu6g/93+wlktP6sKpbjCZs2Z30LIHj1oLxJj6xFogYSj1rgMxULZt7djebY/5Xu3TmvF/lw/xBg+Am08vy3Jz59m9+fCWUYB1YRlT31gLpAqsAeKIjhKW3jWOlITw07hXxZgTyla+XzeqGyJCSkIM2/bnoqrWEjSmnrAWSDhsECRAy+T4Wl2T8e9rh3LDqd29weJwfjFvf7uDVxf7p1DZf6SAnHwbGzEmEiyAhMGm8da90ce3ZsrZvQOOf7hqp/d1cUkpQx6Yx4Cpc9hTA+MxxpiqsQASBltIWH+U+szu3euTAmXoQ/MjUBtjmjYLIGHwzsKKcD2ashtGdwegXVqC99gzn30fqeoYY4hQABGRx0RkvYisFJF3RCTokmQRuVVEVovIGhH5rc/xdBGZKyIb3c8tarO+1gKJvMnje3FC2xQO5xd7j/Ustw4lY/IsVpfb/MoYU3si1QKZC/RT1QHABmBK+QIi0g9n29qhwEBggoh45ndOBuar6nHAfPd9rfGOgVgEiRgRoWVyHIcqmcr7zrc76qhGxpiIBBBVnePuaw6wEOgUpFhvYJGq5rplPwcudM9NBP7tvv43cH5t1rfUbYJEWQCJKFVYsvUg+UVOnqzN+3JpFhvN/RP7esvk5NmMLGPqSn0YA7kWmB3k+GpglIi0FJFEnH3SPelb26qqZzrOLiDkijYRmSQiS0Rkyd69e6tVQevCqh++/mE/AHPWOivVV2dl06F5Aq18Usz/d+n2iNTNmKao1hYSisg8oF2QU3eq6ky3zJ1AMfBK+UKquk5EHgHmAEeB5UBAilZVVREJuVBDVacD0wEyMzOrtaBDvS2Q6lxtakpKfAyHC4qJixay84pYvPkAAMN89hgZ1i09UtUzpsmptQCiquMqOi8iVwMTgLEaYsNrVX0OeM4t/xDg+fNyt4i0V9WdItIe2FNjFQ+itIlvaVtfzJg0nAl/X8CCTfu841E/HdKJFon+K+JttboxdSNSs7DGA3cA56lqbgXl2rifu+CMf7zqnnoPuMp9fRUws/Zqa+qLzi0SAfjPwm38Z+FWAH41pqd3y12ARZsP8PQnmyJWR2OakkiNgTwNpABzRWS5iDwDICIdRORDn3Jvicha4H3gZlU95B6fBvxERDYC49z3tSbG7bsqsZQmEZWSUNZg/nLjPgBSEwIb0X+Zu4GSUuXXry7j8Y+/q7P6GdPURCSZoqr2DHE8C2ew3PN+VIhy+4GxtVO7QFGeAFJqASSSooIMQvkmdBzRvaV3oP3URz9lxyFny90/nHlC3VTQmCamPszCqveixQJIfeWb0PGFa07yvvYED2NM7bEAEoYWSXEAJMVb9vtIa5NSNmX3znLJFhNio4NeszO74mBSXFJqm1UZUw0WQMJw1YiuTD23D78Y0TXSVWny3rn5ZO/rtj55sTw6Nm8WcGzr/pDzNMgtLOb3/13BoPvnehcoGmPCYwEkDDHRUVx9cjdio+2fK9J8A0RrnwWEHr47G3pUNJDe556Pmbk8C8Avz5YxpnL2G9E0OL86rQcAJ2UE5tD883l9A44t2Xow6H3KtzgO28ZUxlSJBRDT4Nwxvhdbpp1DTJAWYVxMFLeHOetqd7lNqKwFYkzVWAAxjc45/dsHHAuW7GD0Y5/5vT9kiRiNqRILIKbR6ZKeyCk9W/HCNSdxwaCOADw4a51fmbzCsu6rx386EICrnl/MzOWWDt6YcFkAMY1OVJTwn+uGMeaENvx0iLNTwLMLNgPOuMdzCzbT+56PvOWHdy9LwHjra8vrtrLGNGC2sME0agM6O5tdtk11Zmz9Z+FWHvBpjfTrmEpqs9ig1xpjKmYBxDRqyfEx9GqXQpf0RAbfP5cD5RYMrt6RQ0q5BaK7c/Jpmxq4xsQY48+6sEyjl5oQy+Z9RwOCh4eIcP/5/bzvPfuMGGMqZgHENAkb9xwJevwBN3BcObwrA93urlcWbWXe2t2W+8yYSlgAMY3e4i2BLYoObhqU7q2TvMee/UUmAAt/OMB1Ly3hqucX100FjWmgLICYJqmTuzlVtM/OhS3dpJkeCzbtC+teJaXKiIfn85btx26aGAsgptG7e0KfgGOK0z3lu/VtVJRwSWanKt//+71H2Jmdz+S3V1a/ksY0QGEFEBHpISLx7uvTROQWEWle3S8qIo+JyHoRWSki74S6l4jcKiKrRWSNiPy2qtcbA/DLU7px+bAu/OGM473Hbj+zFx2bN6NPh1S/sh18kjVGiZMKfsj9c1m9Izvk/X/+r0UAFJUoP+wNPtZiTGMUbgvkLaBERHoC04HOlO1PXh1zgX6qOgDYAEwpX0BE+gHXA0OBgcAE9+uHdb0xvh68oD+/Pv04ALq3SmJot3S+mnw6yeWm8HZIKwsgpQr/+mIz+48WMuHvC0Lee9+RAu/r0//yeQ3X3Jj6K9wAUqqqxcAFwN9V9XYgMOFQmFR1jns/gIVAsH6D3sAiVc11y34OXFiF640JsGrqGXx4a9CdkgH/fdcBDhwtCw7BZmUFy7FlTFMRbgApEpHLgKuAD9xjNbV891pgdpDjq4FRItJSRBJx9krvXIXrARCRSSKyRESW7N27t0YqbBqulITYkDsXAuQX+6d4f9fdKwTgSEFgtt6HPlwXcMym/5qmItwAcg0wAnhQVTeLSDfg5YouEJF57vhF+Y+JPmXuBIqBV8pfr6rrgEeAOcBHwHLA7393Rdf73Ge6qmaqambr1oGbDRnja2SPVgCc2bdtwLmcINl6V/mMjYA4QrAAACAASURBVPRskwzAu99aQkbTNIQVQFR1rareoqozRKQFkKKqj1RyzThV7RfkYyaAiFwNTAAu1xD9AKr6nKoOUdVTgYM44x2Ee70xVdU2NYEt087h/12ZGXAuJ8iGU+P7tvO+9ixK/CbIuhNjGqNwZ2F9JiKpIpIOLAP+JSJPVPeLish44A7gPFUNuWG1iLRxP3fBGf94tSrXG3MsRnRv6fc+2EyswpJSAK4f1Y0BndIAeO2bH2u/csbUA+F2YaWpag7OL/GXVHUYMO4Yvu7TQAowV0SWi8gzACLSQUQ+9Cn3loisBd4HblbVQxVdb0xNumF0d7/3d7+7BnBSwt/+3xXszM7jSH4xUQJTzupNswrGVsp7a+l2MibPYtX20NODjanvws3GGyMi7YFLgDuP9Yuqas8Qx7NwBss974NOlwl1vTE16aSMdC4b2oVrT87gJ3/9wtvamPL2Kt75dgeFJaWkJsSS1iyWqChnQeLo41tzKDd40kZfv//vCgA+WJVFf7flYkxDE24AuQ/4GPhKVb8Rke7AxtqrljGRlxQfw8MX9vc7dt2/v2Heuj2Akwbl8w17aZFYlgKlRWIsP+wLfzFhqc3YMg1YuIPo/1XVAap6k/v+B1W9qHarZkz9ccf4EwC8wQPgaGEx2w7kkuKzIVXzxDgOHQ1/b3W3UWNMgxTuIHonN2XIHvfjLRGxxXumySi/6RTAx2t2A3Dqca28x9KT4jhcUExRBZHhqflljfdgM7uMaSjCHUR/AXgP6OB+vO8eM6ZJiI8JPUCeGFcWXFokOq2RldsPseiH/UHLPzHXOxud/T5pUIxpaMINIK1V9QVVLXY/XgRsVZ5pMmJjyrL2fnnHGJonlnVbLd160Pu6uTsectE/v+Zn0xdWmuokt7CkwvPG1GfhDqLvF5ErgBnu+8uA4H9eGdMIndWvPUcKSrhwUEeS4mM4lFvW9dSxedn+6enl9hTJLyqlWVzw1kvn9GYWQEyDFm4L5FqcKby7gJ3AxcDVtVQnY+qdhNhorhzelSR3LGRYt3Tvud+feYL3tW/LBCC7XPqTjbsPe1+3TUkgtzAwv5YxDUVYLRBV3Qqc53vM3Z/jydqolDH1XUbLJBZtPsAvT+lGakJZ0PCd0gtOAGmXlsCUt1cREyUUl5YNrrdNS2DHobw6q7MxNS3cLqxgfocFENNE3X1uH4Z1T+eCQR39jpcPIIdyC1FVZizeFnCPVklx5BaWoKr86pVl/DSzE6f3CkziaEx9dSxb2krlRYxpnJLjY7hwcCe/LXEBmsU5XV3XnJwBOC2Qo0HGOX5zek+y84rIziviV68sY/bqXVz74pK6qLoxNeZYAogtoTUmiPvP78e1J3cDnAAyd+2ugDK3jTueNVk5AMxeHXjemIagwgAiIodFJCfIx2Gc9SDGmCBS3dXp2XlF3Pb6Cr9zsdFCVJQw9by+kaiaMTWmwgCiqimqmhrkI0VVj2X8xJhGLSU+higJnIUFUFTiNN47t0j0HotxkzG+vyIroLwx9dWxdGEZY0KIihJKFf7+yaaAc5ee5OzM3KVlWQDxpIKfNnt93VTQmBpgAcSYOpDZtYX3dbu0hMDzGc75lslxAeeMqa8iEkBE5DERWS8iK90kjc1DlLvV3Ud9jbvupPz534uIikirYNcbU1/cdFoP72vfjaf+fF5frh6ZwZOXDgKgdXJ8ndfNmOqKVAtkLtBPVQfg7HM+pXwBEekHXA8MBQYCE0Skp8/5zsAZQOAEe2PqgV/5BI1Rx7Xmdz85HoAJA8vmn1w1MoOp5/UlrVkszRNjmb9+Dx/ZrCzTQEQkgKjqHFX15HBYCARLDd8bWKSquW7Zz3G21PX4K86+6Dad2NRLFw4u+7GOi4nilrHHsWXaOXRs3ixoeU9+rTeW2J7qpmGoD2Mg1wKzgxxfDYwSkZYikoiz1W1nABGZCOxQ1RVBrjOmXkhNqNpExYRY57/jJ+v3cPK0T2qjSsbUqFqbiisi84B2QU7dqaoz3TJ3AsXAK+ULqeo6EXkEmAMcBZYDJW4w+RNO91U49ZgETALo0qVLNZ7EmOppkxo4WF6Rnm2SWb3DWVwYLEeWqpKdV+RNGW9MpEll+xXU2hcWuRq4ARirqrlhlH8I2A58CcwHPNd0ArKAoapaYedxZmamLlli6SJM3dmy7ygAGa2SKi37v037+Pmzi7zv19033i8V/P/7/Hsenr2ehVPGBp3JZUxtEZGlqppZ/nikZmGNxxm/OK+i4CEibdzPXXDGP15V1VWq2kZVM1Q1AyeoDK4seBgTCRmtksIKHgAje7aibWrZLKze93xEflFZHq23lm0HYJ/tYmjqiUiNgTwNpABzRWS5iDwDICIdRORDn3JvichanC10b1bVQxGoqzF1ZneOf3DY4LN/yJF8Z95JQXFZUDmcX8TW/UfrpnLGlBORdCSq2jPE8SycwXLP+1Fh3Cuj5mpmTGSlNYv1S3+Sk1e24VRWdj4AX3+/nyFdnQ2txj/5JTsO5fH6pOEM696ybitrmrz6MAvLGONa8Mcx3O6zw+G0j9aRX1TC4s0HvMcen7PB+9oz2P7xmt11V0ljXJYQ0Zh6JCUhlpvH9CQ+JooHZq1j9Y4cRk77hANHC/3KPfP590ybvZ5OLZqx/WAeGa0SQ9zRmNpjLRBj6qHrRnXnuDbJAAHBA8qSLm4/6LRA9h9xyuQXlXDFs4v4Ye+ROqqpacosgBhTTwVLBR+KJ8jMWrmTBZv2cfpfPicnP/zrjakOCyDG1FNJ8YE9zAv+OCZoWU8AWbL1oPfYgKlzyC0sDlremJpgAcSYeurknoGzqkLl0Zq1aidHC4qZsdg/t+jzCzaTMXkWfe/5qFbqaJo2CyDG1FP3TAjc8lZEQpa/7/21Acc8M7aOFpZUqUvMmHBYADGmnoqLiaJDBSlLOjZvxl3n9Pa+f72SLL6etCrG1BQLIMbUY54Wx6RTu/PF7c74x2d/OA2A2b8dxXWjuvPOr0b6XdOjdfDUKXsPWwoUU7MsgBhTj5WUOslOzx3QwbuHekarJLZMO4fUhFgABnVpwdBu6d5r7prQJ+i99h+1AGJqlgUQY+qxs/o7OyJ0D9Gq8GidUpaEMdQ+JOt2HuawTe01NcgCiDH12JSzerP0rnFBp/T66ppethI9PiY6aJkX/7eF/lPn8PX3+2u0jqbpsgBiTD0WFxNFy+T4Sss1iy0LGp1bOMHkPJ+9131d9q+FZEyexbb9uRzKLaS4pLRmKmuaHMuFZUwj4Nl4akT3lqQlxrJl2jms2p7NeyuyQl5z6mOfel9vmXZOrdfRND7WAjGmEfj5sC5cNrQLz1wxxHssMb6sVfLQBf0jUS3TyFkLxJhGIDEuhocv9A8ScdFlfx+elNGirqtkmoBIbWn7mIisF5GVIvKOiDQPUe5WEVktImtE5Lflzv3GvccaEXm0bmpuTMORnhQHwLjebWmTWvEe6qXudGFjqiJSLZC5wBRVLRaRR4ApwB99C4hIP+B6YChQCHwkIh+o6iYRGQNMBAaqaoFn73RjTJmk+Bjv2Iaqf4CIj4mioLhs8PxoYTEpCbEcKSgmKS66wpQpHofzi0iOjwmrrGmcItICUdU5qupJE7oQ6BSkWG9gkarmumU/By50z90ETFPVAvd+e2q7zsY0ZOV/yb94zVC/9zn5xWTnFtHv3o95av6mCu+lqqzbmUP/qXNsJ8Qmrj4Mol8LzA5yfDUwSkRaikgizl7pnd1zx7vnFonI5yJyUqibi8gkEVkiIkv27t1b45U3pqG4/cwTuGxoF165bhgjevhn+s3JK+Kom/r9319vqfA+3aZ8yFl/+xKAD1aGnuVlGr9a68ISkXlAuyCn7lTVmW6ZO4Fi4JXyhVR1ndu9NQc4CiwHStzTMUA6MBw4CXhDRLpr+Xa6c5/pwHSAzMxM6+g1TdbNY3r6vT/1+NZ8scH5oyonz+mOAihyu7YKi0uJjRa/1stn3/k39mOirPuqKau1FoiqjlPVfkE+PMHjamACcHmwX/zuPZ5T1SGqeipwENjgntoOvK2OxUAp0Kq2nsWYxuj/XTGE567KBODlhVvJL3L+PjtcUMzRgmKOv2s2f5230e+aq1/4xu99lAWQJi1Ss7DGA3cA56lqbgXl2rifu+CMf7zqnnoXGOOeOx6IA/bVZp2NaWyaxUXTsYWzQdUHK3fys+kLvec82+E+NX9j0Gs9ikusUd+URWoM5GkgBZgrIstF5BkAEekgIh/6lHtLRNYC7wM3q+oh9/jzQHcRWQ28BlwVqhVjjAnthLYpRLutCM+2uOAfOLIO5YW8fndOPgB5hSVMfmul3z1M4xeRabyq2jPE8SycwXLP+1EhyhUCV9RO7YxpOkSEKCkbXPSYsbhsc6qR0z7xS3WSEBtFfpEzTuLZ5fD9FVm89s2PREWJrXpvQurDLCxjTAQ9/tOBYZeNj4niqhEZXDm8KwDrdx0mt7CYdbtyAHh10TZvq8Q0fhZAjGniTjuhbB3uIxcFbz2UliqqSkFxKfExUdx/fj+Ob5sMwKY9R3jhqy3esr959dtara+pPyyAGNPE+W5A1bdDWtAy+44WUOimfY93U8ffN7EfALtz/Hc6XLzlQG1U09RDFkCMaeJ813kc57YqAG4Y3d37+uPVu7ypT+JjnF8bnlxb17+0pMpf82/zNnLZ9IUBKVZMw2IBxBjj5bub4e1nnOB9PXv1LgqK/ANI20oSNFbkr/M28PUP+3luweZq38NEngUQYww/PHQ2mx48y+9YTHQUGS2d3Q037D7C2L98BpQFmbRmsTRPjA16v3Cz+z4wa101a2zqAwsgxhiiooQYd/+Qv/x0IE9c4szM+uz2MZzYuTn7jxaQk+/kyoqPLfu10SalbLvd8X3LMhfNX2/5TZsCCyDGGD8XDenEhYPLEmS3SIzFd6jCt5urQ3NnJfsbN4zgmSuHeKcEPzBrbdB7FxSXeNeOeEybvZ65a8PL6rsmK5sPV+3k0+/2MOXtVbaPSYTZjoTGmAp9+p1/FuuikrJ9RG4c3YOMlkkM6erseHhm37b84b+wdX/wDEXXv7TUm8DR45nPvwcq35c9Y/KsgGNfbtzLgj+eXvlDmFphLRBjTIV+/5Pj/d7nFZatWx/evSVTz+vrTYeSkhB8TMTDN3iUz8PoG5jCtf1g6DQrpvZZADHGVOjcgR0AuHXscUw5qxfnD+pYrfuUn7I78+ZT/N4fd+ds9h3xX1MSjhLrxooYCyDGmApltEri/V+fwo2je3DD6B7ExVT8a+OG0d2Ji4lCVZm1cicn3DWbguISbvrPMr9yvdunBFx762uhV7F3Tm8W9HhuYXHQ46b2WQAxxlSqf6c0msVFV14QaN4sjsLiUvKLSrn51WUUFJcyZ81uPlqzy1smo2Wid9aXr4q6pH484H9uXG8nBYtvl5qpWxZAjDE1Kq2ZMw7iO9vKNyX8WzeNZM5towH4/PbTGNot3XvuaEFgMHj56y0BA+gr7j2Ds/u3B2BOmDO4TM2zAGKMqVFJ8U5L5eWFW7zH1mTleF8P7JTm7Qbr2jKJnw/t4j0XbAzk7plrvK+vH9WNt24aSVqzWBLdFtFd766uUv32HSlgd04+Ix6ez1tLt1fpWuPPAogxpkbFuV1T//j0e++x91ZkAfC3S08M6Lryzb8FsL+CgfRRx7X2ThlOjCtbhfC/TeFtSLp6RzaZD8xj2EPz2Zmdz/0h1quY8ERqS9vHRGS9iKwUkXdEpHmIcreKyGoRWSMiv/U5fqKILHR3M1wiIkPrrvbGmIoMyWgR8tzJPVsFHOvbIY0XrjmJU9xzd7y50nvuxwP+60kSfcZhYn0C0c+fXRRW3Z7+ZJPf+0O5RSFKmnBEqgUyF+inqgOADcCU8gVEpB9wPTAUGAhMEBHPToaPAn9W1ROBe9z3xph6oHVyvDfhYnmtkuODHh9zQhtuOq0HAEnxZS2LUY9+6lfOdyC/R5ukKtVr2baDfgP55thFJICo6hxV9cy9Wwh0ClKsN7BIVXPdsp8DF3puAaS6r9OArNqsrzEmfCJCqbvm46LBwf5rBzeie0vA6e5699sdQcsk+XRbtUlJ4NXrhoV9/wv/739hlzXhqQ9jINcCs4McXw2MEpGWIpKIs1d6Z/fcb4HHRORH4HGCtGA8RGSS2821ZO/evaGKGWNqUItEZ6+QCQPac+vY48K6JspnafpvX1/OPz7dFFCmdYp/C2ZEj5be1+VTpJjaV2sBRETmueMX5T8m+pS5EygGXil/vaquAx4B5gAfAcsBzxy/m4DbVLUzcBvwXKh6qOp0Vc1U1czWrVvX2PMZY0JLdruhEmKjvQEkKcx1JB6Pffyd93V6Uhxv3jjCr3sLnNbO79xUK794fjFff7+/ynV9bfE2cvKLyC+y9SRVVWsBRFXHqWq/IB8zAUTkamACcLmG2JZMVZ9T1SGqeipwEGe8BOAq4G339X9xxkmMMfWEJzdWs7hooqKEmTefzKe3n1bpdfee2yfo8Q7NE8jMSA96rmebsllcn2/YG1YgWDhlLL88pRsAk99exYCpczj/H19Vel1de/nrLRzKLYx0NUKK1Cys8cAdwHmqGjxtp1Oujfu5C874x6vuqSxgtPv6dGBj7dXWGFNVngASG+18Hti5OW1SKt/B8JqTu9EsNrClUlgcOtGi78D6M59/z5/fX0tpqTJz+Y6gq9T/fF5f2qUlBNxz/a7Dx9wKKS4p5Z1vt9dImvkf9h7h7plrGDntkypdV1RSyh/fXMm2EBmRa1KkxkCeBlKAue5U3GcARKSDiHzoU+4tEVkLvA/crKqH3OPXA38RkRXAQ8CkOqy7MaYSngBSnS3PX79huN/7S0/qzP9dPjhk+eIS/y/y3yU/Mn/9Hm59bTnPf+VsmXvlc2XTfK8amQHABYMDk0L+/ZPgf4tu2XeU7DCm/M5YvI3bXl/h/brHItcNfrmFJew9HH6SyU17jvD6kh+58T9Lj7kOlYnULKyeqtpZVU90P250j2ep6tk+5Uapah9VHaiq832OL3C7tgaq6jBVrf1/KWNM2G4c7UzJ7ZyeWOVrB3RqzqMXDfC+n3peX3q2CUy86NE+zb9lU1yqXP/SEsAZR8krLOHLjYELDQd3aUHv9ql+xz5Zv5dd2fkBZU97/DPO+tsX3vdFJaUBG2MB5LktmJrYqnf7wbIWxEkPzuP9FeFNNvUE1P1Hq57ZuKrqwywsY0wjc+7ADmyZdo43L1ZVXXJSZ+9+IaHWlHj065jG4j+N5cTOQdcj8+yXP5TdN9N/WvFF5Voh63bmMPzh+QSTlZ3PX+duYPWObH7z6rcM/POcgDLpSWWzxF5ZtJWrnl9cYd2DWbn9EPfMXB2QPPI3M75l/a6cEFeVyS92gtjunIJaT3VvOxIaY+qleb8bTdahfESk0rJtUhO4aEgnlv94KOCcpzvp/BM78OjFA/3Oje3dttLWgu9GV3+bv5FnPv+eAnf8pLC41C+9vWfs44S2Kdz5TtVydHk89vF3fLlxH+N6tyEmSij2CQJb9+fSq11qBVf7Zyc+lFtIyxCLN2uCtUCMMfVS99bJnHJcYOqTUC47qbPf+/QkZy3KQXfsYvO+owHX+A7Ye1bCl3e0wH+/kQKfwffD+f7dWHfPdILGd7sPe49VNd28J33Lpj1HaJuawGknlC0/KClVDucXkTF5FpPfWhn0+jyfiQB7qjB2Uh0WQIwxjUL5JI0x5fbMneYzruLRxmdhYqj1Kofz/QOIb4uj/LmCILPFNu5xgsmmPUd4ePa6SmdoeVpcW/bn0r11EtE+LbBtB3LpP9XpOnvtmx+DXu87k2zqe2uClqkpFkCMMY3G/RP7el9POrU7o3xaMMEG9H1XvyfERnNO//YcLSxh2baD3uPlB8s7Ni/bGTEnv/KZWTe87Mzx+eNbK/l/n//Apr1HKizv22PXqUWiXx3fXlZ5+vmFPxzwvu7XMa3S8sfCAogxptEY1r0stUnXlkne1PIQeiX83RP68NxVmQCs3ekMUj/z2fccKSgmO6+Im17xn+QZ7fML3bcF8ri7cr5cw4c2qQks3XqApVsPutdUHHRio8rqfPGQTtx1Tm9O79WG2GgJWLuiqmzbn+vXzTZj8Tbv65bJcby6aJvfNOaaZIPoxphGw3dMIzuvyC/1SajBeM+KdCjbDKuwpJSTHpjnN57gceBo2crwHJ/WydNBcncBnNgpjYv++bXPNRXv4e4boI5vm0xKQizPX30Sp//lM7aUG8d5cNY6nl2wmQ5pCfxvytiAex3OL+bRj5zAtnjzAb/dH2uCtUCMMY1GpxZl3Usd0hL89g8Jx7kDOgBOYPAEjwkD2vuV8Q0gS7aWdXUN7+78cj53YAeuGN6FlPgYOjZvxr+/3up3/TUvfsORgtBBJLew7Jxv9uF2qQmUHz55doEzwywryNoVgH9+Vrap187s0PvNV5cFEGNMo+HbyhjZs5VfmpNwTDq1OwDLtpVNB/5g5U6/rrBQ2qU6CxofuqA/D5zfn1V/PjPkCvIDRwLzW2UdyuOC//uKbT6baPmOf3juDwTNcPzD3iN+wcfXqONaMfHEwJX3x8oCiDGmUZlx/XD+fa2TX3WY22VznU83VUVEJCBlPDhdWsvv+QkPX9jf7/gRdwxk4+7DfL/3KL3bp/p1mxWWBM/hFawFMnLaJ3y77RClCuf0b8/0K4f4nfcdfD8pSGLJmcuzmLncWa1+w6ndaZFYtoizolxix8LGQIwxjYrvHiHj+7XnyzvGVCmlSqhur+aJcX6tAIDDBc4YyE/+6qQ5OamC7Xx9VTZ7a2i3dM7o287vmO8WvsGC3N/ml+XxOq5tCl/+8XT63fsxALeEuSdLVVkLxBjTqFU1H5fvQPwVw7v4nWvrE0AGdkoLWAeSGBf8b/JJp3Zn/u9Hk9nVCTDB8mj161i2wjw5PvA+vz/jeO9r34H2uycEpsAvKC7xu0ewvehrgrVAjDHGx48+YxAPnN+f28/s5V2U2KtdCqkJMYw+oQ3b9h/ly437OOgzqP55iF0R/3R2bwD++rMTGfXop36zt8DpYlq9oyzPVXJC4K/mkT1aMeP64ezKyaOLT1As3yoC2H7QGTB/5bphfutWapoFEGOM8XG0XOoR34SQUVHCyqlnApAxeRYAg+6fG/Jer143zLv+AyA1wbmXbwvkoQ/XMf2LH/yuSwnSAgH/7rnmibEcyi0iMS6a9KQ4v9lhI91ytdXy8LAuLGOM8eGbe6oi155c+cD8yJ6t+I3P+ENyQgwi/gGkfPAAArbuDcYzJpIQG83NY3p6jw/tls6o4+pm+24LIMYY42Pl9mwAftKnbYXlgk2lfeaKIUFKlomOEtqmJLDTXbexaY9/WhNPbq5gXVjleaYWJ8ZFc3qvNiTFRfPBb07hjRtGVHptTYlYABGR+0Vkpbsj4RwR6RCi3FUistH9uMrn+BARWSUim0TkKQkn57MxxlSiR+skAC7J7FxhubRE/71Orj25W6VBB6BdWgLvr8iitFQZ98TnfufuPbcvbVPjwxq38GwXnBAbTbdWSay5b3yt574qL5ItkMdUdYCqngh8ANxTvoCIpAP3AsOAocC9IuKZJ/dPnK1tj3M/xtdJrY0xjdovRmQAMKhL8A2qfE0915kBlZIQwz3n9vGbHRWKqlJQXMqd7wbuF3LOgPYs+tM4EoLsC1+epwsrjC9ZayI2iK6qvltrJQHBchyfCcxV1QMAIjIXGC8inwGpqrrQPf4ScD4wu1YrbYxp9M4d2IFzBwbtEAlw9cndiIoSBncJb/0HQGZGOiu2Z/slPYSyYFSVej4xdwPNE+OqdF1NiugsLBF5EPgFkA2MCVKkI+Cb9H67e6yj+7r88WBfYxIwCaBLly7BihhjTLV5Wizh+m7XYb/3l57Umfsm9vPbZyQcvx7TkyuHd6VFUuQCSK12YYnIPBFZHeRjIoCq3qmqnYFXgF/XRh1UdbqqZqpqZuvWdTMzwRhjQrnPZ88SgDP6tq1y8ABnSnEkgwfUcgtEVceFWfQV4EOc8Q5fO4DTfN53Aj5zj3cqd3xHtSppjDF1qHvrZL/3vutMGppIzsLynQM3EVgfpNjHwBki0sIdPD8D+FhVdwI5IjLcnX31C2BmrVfaGGNqWMukwLxWDUUkZ2FNc7uzVuIEhlsBRCRTRJ4FcAfP7we+cT/u8wyoA78CngU2Ad9jA+jGmAZi0Z/KNn/KaJUUwZocG1GteIP3xiQzM1OXLFkS6WoYYwyvf7ONLulJfulJ6isRWaqqmeWPWy4sY4yJgJ+d1PBnhVoqE2OMMdViAcQYY0y1WAAxxhhTLRZAjDHGVIsFEGOMMdViAcQYY0y1WAAxxhhTLRZAjDHGVEuTWokuInuBrdW8vBWwrwarU9/Y8zVs9nwNW31/vq6qGpDOvEkFkGMhIkuCLeVvLOz5GjZ7voatoT6fdWEZY4ypFgsgxhhjqsUCSPimR7oCtcyer2Gz52vYGuTz2RiIMcaYarEWiDHGmGqxAGKMMaZaLICEQUTGi8h3IrJJRCZHuj7VJSJbRGSViCwXkSXusXQRmSsiG93PLdzjIiJPuc+8UkQGR7b2gUTkeRHZIyKrfY5V+XlE5Cq3/EYRuSoSzxJMiOebKiI73O/hchE52+fcFPf5vhORM32O17ufXxHpLCKfishaEVkjIp4trRvF96+C52sU3z8vVbWPCj6AaJw917sDccAKoE+k61XNZ9kCtCp37FFgsvt6MvCI+/psnH3mBRgOLIp0/YM8z6nAYGB1dZ8HSAd+cD+3cF+3iPSzVfB8U4E/BCnbx/3ZjAe6uT+z0fX15xdoDwx2X6cAG9xnaBTfvwqer1F8/zwf1gKp3FBgk6r+oKqFwGvAxAjXqSZNBP7tvv43cL7P8ZfUsRBoLiLtI1HBUFT1C+BAucNVfZ4zgbmqekBVDwJzgfG1X/vKhXi+UCYCrVNYWAAABjhJREFUr6lqgapuBjbh/OzWy59fVd2pqsvc14eBdUBHGsn3r4LnC6VBff88LIBUriPwo8/77VT8g1CfKTBHRJaKyCT3WFtV3em+3gW0dV831Oeu6vM0xOf8tduN87yni4cG/HwikgEMAhbRCL9/5Z4PGtH3zwJI03KKqg4GzgJuFpFTfU+q05ZuNPO6G9vzuP4J9ABOBHYCf4lsdY6NiCQDbwG/VdUc33ON4fsX5Pka1ffPAkjldgCdfd53co81OKq6w/28B3gHp3m829M15X7e4xZvqM9d1edpUM+pqrtVtURVS4F/4XwPoQE+n4jE4vxyfUVV33YPN5rvX7Dna0zfP7AAEo5vgONEpJuIxAGXAu9FuE5VJiJJIpLieQ2cAazGeRbPzJWrgJnu6/eAX7izX4YD2T5dC/VZVZ/nY+AMEWnhdiec4R6rl8qNQ12A8z0E5/kuFZF4EekGHAcspp7+/IqIAM8B61T1CZ9TjeL7F+r5Gsv3zyvSo/gN4QNnBsgGnNkQd0a6PtV8hu44MzhWAGs8zwG0BOYDG4F5QLp7XIB/uM+8CsiM9DMEeaYZON0ARTh9w7+szvMA1+IMWm4Cron0c1XyfC+79V+J84ukvU/5O93n+w44qz7//AKn4HRPrQSWux9nN5bvXwXP1yi+f54PS2VijDGmWqwLyxhjTLVYADHGGFMtFkCMMcZUiwUQY4wx1WIBxBhjTLVYADGNioiUuFlOV4jIMhEZWUn55iLyqzDu+5mIZNZcTRs+EXlRRC6OdD1M5FgAMY1NnqqeqKoDgSnAw5WUbw5UGkAiRURiIl0HY0KxAGIas1TgIDg5iURkvtsqWSUinoym04AebqvlMbfsH90yK0Rkms/9fioii0Vkg4iMcstGi8hjIvKNmyDvBvd4exH5wr3vak95X+Lsz/Ko+7UWi0hP9/iLIvKMiCwCHhVnj4x33fsvFJEBPs/0gnv9ShG5yD1+hoh87T7rf918TIjINHH2p1gpIo+7x37q1m+FiHxRyTOJiDwtzt4U84A2NfnNMg2P/XVjGptmIrIcSMDZk+F093g+cIGq5ohIK2ChiLyHs+dEP1U9EUBEzsJJlz1MVXNFJN3n3jGqOlScTYDuBcbhrA7PVtWTRCQe+EpE5gAXAh+r6oMiEg0khqhvtqr2F5FfAE8CE9zjnYCRqloiIn8HvlXV80XkdOAlnGR8d3uud+vewn22u4BxqnpURP4I/E5E/oGTOqOXqqqINHe/zj3Amaq6w+dYqGcaBJyAs3dFW2At8HxY3xXTKFkAMY1Nnk8wGAG8JCL9cFJhPCROBuJSnJTYbYNcPw54QVVzAVTVdz8OT8K/pUCG+/oMYIDPWEAaTh6jb4DnxUmo966qLg9R3xk+n//qc/y/qlrivj4FuMitzyci0lJEUt26Xuq5QFUPisgEnF/wXznpmIgDvgaycYLocyLyAfCBe9lXwIsi8obP84V6plOBGW69skTkkxDPZJoICyCm0VLVr92/yFvj5BNqDQxR1SIR2YLTSqmKAvdzCWX/dwT4jaoGJPBzg9U5OL+gn1DVl4JVM8Tro1Wsm/fL4mywdFmQ+gwFxgIXA78GTlfVG0VkmFvPpSIyJNQzic/2q8aAjYGYRkxEeuFsCbof56/oPW7wGAN0dYsdxtly1GMucI2IJLr38O3CCuZj4Ca3pYGIHC9O5uOuwG5V/RfwLM7WtMH8zOfz1yHKfAlc7t7/NGCfOntLzAVu9nneFsBC4GSf8ZQkt07JQJqqfgjcBgx0z/dQ1UWqeg+wFyd1eNBnAr4AfuaOkbQHxlTyb2MaOWuBmMbGMwYCzl/SV7njCK8A74vIKmAJsB5AVfeLyFcishqYraq3i8iJwBIRKQQ+BP5Uwdd7Fqc7a5k4fUZ7cbZhPQ24XUSKgCPAL0Jc30JEVuK0bgJaDa6pON1hK4FcytKdPwD8w617CfBnVX1bRK4GZrjjF+CMiRwGZopIgvvv8jv33GMicpx7bP7/b+cOjQAEYiAApij6phcEBaDpIS8e++bMC3YrSNRNkpnU/NZ8LXo6a96U7qp6ah14/IRvvLDJt0Y7uvvdXQskrLAAiJhAAIiYQACICBAAIgIEgIgAASAiQACIDFTGH44WYpLaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2019-07-21T07:10:54.182Z"
    }
   },
   "outputs": [],
   "source": [
    "learner.recorder.plot_metrics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.fit(10,1e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:59:02.948812Z",
     "start_time": "2019-07-26T06:59:02.933485Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 AtomTransformer(\n",
      "  (transformer): Transformer(\n",
      "    (drop_emb): Dropout(p=0.0)\n",
      "    (layers): ModuleList(\n",
      "      (0): DecoderLayer(\n",
      "        (mhra): MultiHeadAttention(\n",
      "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (1): DecoderLayer(\n",
      "        (mhra): MultiHeadAttention(\n",
      "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (2): DecoderLayer(\n",
      "        (mhra): MultiHeadAttention(\n",
      "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (3): DecoderLayer(\n",
      "        (mhra): MultiHeadAttention(\n",
      "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (4): DecoderLayer(\n",
      "        (mhra): MultiHeadAttention(\n",
      "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (5): DecoderLayer(\n",
      "        (mhra): MultiHeadAttention(\n",
      "          (attention1): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention3): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (attention4): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([1024]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (scalar): Conv1d(1033, 4, kernel_size=(1,), stride=(1,))\n",
      "  (magnetic): Conv1d(1024, 9, kernel_size=(1,), stride=(1,))\n",
      "  (dipole): Linear(in_features=29696, out_features=3, bias=True)\n",
      "  (potential): Linear(in_features=29696, out_features=1, bias=True)\n",
      "  (type_embedding): Embedding(9, 506)\n",
      "  (atom_embedding): Embedding(6, 512)\n",
      ")\n",
      "1 AtomTransformerOld(\n",
      "  (transformer): Transformer(\n",
      "    (drop_emb): Dropout(p=0.0)\n",
      "    (layers): ModuleList(\n",
      "      (0): DecoderLayer(\n",
      "        (mhra): MultiHeadAttentionOld(\n",
      "          (attention): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (1): DecoderLayer(\n",
      "        (mhra): MultiHeadAttentionOld(\n",
      "          (attention): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (2): DecoderLayer(\n",
      "        (mhra): MultiHeadAttentionOld(\n",
      "          (attention): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (3): DecoderLayer(\n",
      "        (mhra): MultiHeadAttentionOld(\n",
      "          (attention): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (4): DecoderLayer(\n",
      "        (mhra): MultiHeadAttentionOld(\n",
      "          (attention): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (5): DecoderLayer(\n",
      "        (mhra): MultiHeadAttentionOld(\n",
      "          (attention): Linear(in_features=512, out_features=1536, bias=True)\n",
      "          (out): Linear(in_features=512, out_features=512, bias=True)\n",
      "          (drop_att): Dropout(p=0.0)\n",
      "          (drop_res): Dropout(p=0.0)\n",
      "          (ln): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (ff): SequentialEx(\n",
      "          (layers): ModuleList(\n",
      "            (0): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (1): ReLU(inplace)\n",
      "            (2): Dropout(p=0.0)\n",
      "            (3): Linear(in_features=2048, out_features=512, bias=True)\n",
      "            (4): Dropout(p=0.0)\n",
      "            (5): MergeLayer()\n",
      "            (6): LayerNorm(torch.Size([512]), eps=1e-05, elementwise_affine=True)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (scalar): Conv1d(512, 4, kernel_size=(1,), stride=(1,))\n",
      "  (magnetic): Conv1d(512, 9, kernel_size=(1,), stride=(1,))\n",
      "  (dipole): Linear(in_features=14848, out_features=3, bias=True)\n",
      "  (potential): Linear(in_features=14848, out_features=1, bias=True)\n",
      "  (type_embedding): Embedding(9, 250)\n",
      "  (atom_embedding): Embedding(6, 256)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "ct = 0\n",
    "for child in learner.model.children():\n",
    "    print(ct, child)\n",
    "    ct += 1\n",
    "    if ct < 13:\n",
    "       \n",
    "        for name, param in learner.model.named_parameters():\n",
    "\n",
    "            if 'scalar' not in name and name != 'w1' and 'layers.5' not in name:\n",
    "                param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T06:58:03.504683Z",
     "start_time": "2019-07-26T06:58:03.462404Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w1\n",
      "net1.transformer.layers.0.mhra.attention1.weight\n",
      "net1.transformer.layers.0.mhra.attention1.bias\n",
      "net1.transformer.layers.0.mhra.attention2.weight\n",
      "net1.transformer.layers.0.mhra.attention2.bias\n",
      "net1.transformer.layers.0.mhra.attention3.weight\n",
      "net1.transformer.layers.0.mhra.attention3.bias\n",
      "net1.transformer.layers.0.mhra.attention4.weight\n",
      "net1.transformer.layers.0.mhra.attention4.bias\n",
      "net1.transformer.layers.0.mhra.out.weight\n",
      "net1.transformer.layers.0.mhra.out.bias\n",
      "net1.transformer.layers.0.mhra.ln.weight\n",
      "net1.transformer.layers.0.mhra.ln.bias\n",
      "net1.transformer.layers.0.ff.layers.0.weight\n",
      "net1.transformer.layers.0.ff.layers.0.bias\n",
      "net1.transformer.layers.0.ff.layers.3.weight\n",
      "net1.transformer.layers.0.ff.layers.3.bias\n",
      "net1.transformer.layers.0.ff.layers.6.weight\n",
      "net1.transformer.layers.0.ff.layers.6.bias\n",
      "net1.transformer.layers.1.mhra.attention1.weight\n",
      "net1.transformer.layers.1.mhra.attention1.bias\n",
      "net1.transformer.layers.1.mhra.attention2.weight\n",
      "net1.transformer.layers.1.mhra.attention2.bias\n",
      "net1.transformer.layers.1.mhra.attention3.weight\n",
      "net1.transformer.layers.1.mhra.attention3.bias\n",
      "net1.transformer.layers.1.mhra.attention4.weight\n",
      "net1.transformer.layers.1.mhra.attention4.bias\n",
      "net1.transformer.layers.1.mhra.out.weight\n",
      "net1.transformer.layers.1.mhra.out.bias\n",
      "net1.transformer.layers.1.mhra.ln.weight\n",
      "net1.transformer.layers.1.mhra.ln.bias\n",
      "net1.transformer.layers.1.ff.layers.0.weight\n",
      "net1.transformer.layers.1.ff.layers.0.bias\n",
      "net1.transformer.layers.1.ff.layers.3.weight\n",
      "net1.transformer.layers.1.ff.layers.3.bias\n",
      "net1.transformer.layers.1.ff.layers.6.weight\n",
      "net1.transformer.layers.1.ff.layers.6.bias\n",
      "net1.transformer.layers.2.mhra.attention1.weight\n",
      "net1.transformer.layers.2.mhra.attention1.bias\n",
      "net1.transformer.layers.2.mhra.attention2.weight\n",
      "net1.transformer.layers.2.mhra.attention2.bias\n",
      "net1.transformer.layers.2.mhra.attention3.weight\n",
      "net1.transformer.layers.2.mhra.attention3.bias\n",
      "net1.transformer.layers.2.mhra.attention4.weight\n",
      "net1.transformer.layers.2.mhra.attention4.bias\n",
      "net1.transformer.layers.2.mhra.out.weight\n",
      "net1.transformer.layers.2.mhra.out.bias\n",
      "net1.transformer.layers.2.mhra.ln.weight\n",
      "net1.transformer.layers.2.mhra.ln.bias\n",
      "net1.transformer.layers.2.ff.layers.0.weight\n",
      "net1.transformer.layers.2.ff.layers.0.bias\n",
      "net1.transformer.layers.2.ff.layers.3.weight\n",
      "net1.transformer.layers.2.ff.layers.3.bias\n",
      "net1.transformer.layers.2.ff.layers.6.weight\n",
      "net1.transformer.layers.2.ff.layers.6.bias\n",
      "net1.transformer.layers.3.mhra.attention1.weight\n",
      "net1.transformer.layers.3.mhra.attention1.bias\n",
      "net1.transformer.layers.3.mhra.attention2.weight\n",
      "net1.transformer.layers.3.mhra.attention2.bias\n",
      "net1.transformer.layers.3.mhra.attention3.weight\n",
      "net1.transformer.layers.3.mhra.attention3.bias\n",
      "net1.transformer.layers.3.mhra.attention4.weight\n",
      "net1.transformer.layers.3.mhra.attention4.bias\n",
      "net1.transformer.layers.3.mhra.out.weight\n",
      "net1.transformer.layers.3.mhra.out.bias\n",
      "net1.transformer.layers.3.mhra.ln.weight\n",
      "net1.transformer.layers.3.mhra.ln.bias\n",
      "net1.transformer.layers.3.ff.layers.0.weight\n",
      "net1.transformer.layers.3.ff.layers.0.bias\n",
      "net1.transformer.layers.3.ff.layers.3.weight\n",
      "net1.transformer.layers.3.ff.layers.3.bias\n",
      "net1.transformer.layers.3.ff.layers.6.weight\n",
      "net1.transformer.layers.3.ff.layers.6.bias\n",
      "net1.transformer.layers.4.mhra.attention1.weight\n",
      "net1.transformer.layers.4.mhra.attention1.bias\n",
      "net1.transformer.layers.4.mhra.attention2.weight\n",
      "net1.transformer.layers.4.mhra.attention2.bias\n",
      "net1.transformer.layers.4.mhra.attention3.weight\n",
      "net1.transformer.layers.4.mhra.attention3.bias\n",
      "net1.transformer.layers.4.mhra.attention4.weight\n",
      "net1.transformer.layers.4.mhra.attention4.bias\n",
      "net1.transformer.layers.4.mhra.out.weight\n",
      "net1.transformer.layers.4.mhra.out.bias\n",
      "net1.transformer.layers.4.mhra.ln.weight\n",
      "net1.transformer.layers.4.mhra.ln.bias\n",
      "net1.transformer.layers.4.ff.layers.0.weight\n",
      "net1.transformer.layers.4.ff.layers.0.bias\n",
      "net1.transformer.layers.4.ff.layers.3.weight\n",
      "net1.transformer.layers.4.ff.layers.3.bias\n",
      "net1.transformer.layers.4.ff.layers.6.weight\n",
      "net1.transformer.layers.4.ff.layers.6.bias\n",
      "net1.transformer.layers.5.mhra.attention1.weight\n",
      "net1.transformer.layers.5.mhra.attention1.bias\n",
      "net1.transformer.layers.5.mhra.attention2.weight\n",
      "net1.transformer.layers.5.mhra.attention2.bias\n",
      "net1.transformer.layers.5.mhra.attention3.weight\n",
      "net1.transformer.layers.5.mhra.attention3.bias\n",
      "net1.transformer.layers.5.mhra.attention4.weight\n",
      "net1.transformer.layers.5.mhra.attention4.bias\n",
      "net1.transformer.layers.5.mhra.out.weight\n",
      "net1.transformer.layers.5.mhra.out.bias\n",
      "net1.transformer.layers.5.mhra.ln.weight\n",
      "net1.transformer.layers.5.mhra.ln.bias\n",
      "net1.transformer.layers.5.ff.layers.0.weight\n",
      "net1.transformer.layers.5.ff.layers.0.bias\n",
      "net1.transformer.layers.5.ff.layers.3.weight\n",
      "net1.transformer.layers.5.ff.layers.3.bias\n",
      "net1.transformer.layers.5.ff.layers.6.weight\n",
      "net1.transformer.layers.5.ff.layers.6.bias\n",
      "net1.scalar.weight\n",
      "net1.scalar.bias\n",
      "net1.magnetic.weight\n",
      "net1.magnetic.bias\n",
      "net1.dipole.weight\n",
      "net1.dipole.bias\n",
      "net1.potential.weight\n",
      "net1.potential.bias\n",
      "net1.type_embedding.weight\n",
      "net1.atom_embedding.weight\n",
      "net2.transformer.layers.0.mhra.attention.weight\n",
      "net2.transformer.layers.0.mhra.attention.bias\n",
      "net2.transformer.layers.0.mhra.out.weight\n",
      "net2.transformer.layers.0.mhra.out.bias\n",
      "net2.transformer.layers.0.mhra.ln.weight\n",
      "net2.transformer.layers.0.mhra.ln.bias\n",
      "net2.transformer.layers.0.ff.layers.0.weight\n",
      "net2.transformer.layers.0.ff.layers.0.bias\n",
      "net2.transformer.layers.0.ff.layers.3.weight\n",
      "net2.transformer.layers.0.ff.layers.3.bias\n",
      "net2.transformer.layers.0.ff.layers.6.weight\n",
      "net2.transformer.layers.0.ff.layers.6.bias\n",
      "net2.transformer.layers.1.mhra.attention.weight\n",
      "net2.transformer.layers.1.mhra.attention.bias\n",
      "net2.transformer.layers.1.mhra.out.weight\n",
      "net2.transformer.layers.1.mhra.out.bias\n",
      "net2.transformer.layers.1.mhra.ln.weight\n",
      "net2.transformer.layers.1.mhra.ln.bias\n",
      "net2.transformer.layers.1.ff.layers.0.weight\n",
      "net2.transformer.layers.1.ff.layers.0.bias\n",
      "net2.transformer.layers.1.ff.layers.3.weight\n",
      "net2.transformer.layers.1.ff.layers.3.bias\n",
      "net2.transformer.layers.1.ff.layers.6.weight\n",
      "net2.transformer.layers.1.ff.layers.6.bias\n",
      "net2.transformer.layers.2.mhra.attention.weight\n",
      "net2.transformer.layers.2.mhra.attention.bias\n",
      "net2.transformer.layers.2.mhra.out.weight\n",
      "net2.transformer.layers.2.mhra.out.bias\n",
      "net2.transformer.layers.2.mhra.ln.weight\n",
      "net2.transformer.layers.2.mhra.ln.bias\n",
      "net2.transformer.layers.2.ff.layers.0.weight\n",
      "net2.transformer.layers.2.ff.layers.0.bias\n",
      "net2.transformer.layers.2.ff.layers.3.weight\n",
      "net2.transformer.layers.2.ff.layers.3.bias\n",
      "net2.transformer.layers.2.ff.layers.6.weight\n",
      "net2.transformer.layers.2.ff.layers.6.bias\n",
      "net2.transformer.layers.3.mhra.attention.weight\n",
      "net2.transformer.layers.3.mhra.attention.bias\n",
      "net2.transformer.layers.3.mhra.out.weight\n",
      "net2.transformer.layers.3.mhra.out.bias\n",
      "net2.transformer.layers.3.mhra.ln.weight\n",
      "net2.transformer.layers.3.mhra.ln.bias\n",
      "net2.transformer.layers.3.ff.layers.0.weight\n",
      "net2.transformer.layers.3.ff.layers.0.bias\n",
      "net2.transformer.layers.3.ff.layers.3.weight\n",
      "net2.transformer.layers.3.ff.layers.3.bias\n",
      "net2.transformer.layers.3.ff.layers.6.weight\n",
      "net2.transformer.layers.3.ff.layers.6.bias\n",
      "net2.transformer.layers.4.mhra.attention.weight\n",
      "net2.transformer.layers.4.mhra.attention.bias\n",
      "net2.transformer.layers.4.mhra.out.weight\n",
      "net2.transformer.layers.4.mhra.out.bias\n",
      "net2.transformer.layers.4.mhra.ln.weight\n",
      "net2.transformer.layers.4.mhra.ln.bias\n",
      "net2.transformer.layers.4.ff.layers.0.weight\n",
      "net2.transformer.layers.4.ff.layers.0.bias\n",
      "net2.transformer.layers.4.ff.layers.3.weight\n",
      "net2.transformer.layers.4.ff.layers.3.bias\n",
      "net2.transformer.layers.4.ff.layers.6.weight\n",
      "net2.transformer.layers.4.ff.layers.6.bias\n",
      "net2.transformer.layers.5.mhra.attention.weight\n",
      "net2.transformer.layers.5.mhra.attention.bias\n",
      "net2.transformer.layers.5.mhra.out.weight\n",
      "net2.transformer.layers.5.mhra.out.bias\n",
      "net2.transformer.layers.5.mhra.ln.weight\n",
      "net2.transformer.layers.5.mhra.ln.bias\n",
      "net2.transformer.layers.5.ff.layers.0.weight\n",
      "net2.transformer.layers.5.ff.layers.0.bias\n",
      "net2.transformer.layers.5.ff.layers.3.weight\n",
      "net2.transformer.layers.5.ff.layers.3.bias\n",
      "net2.transformer.layers.5.ff.layers.6.weight\n",
      "net2.transformer.layers.5.ff.layers.6.bias\n",
      "net2.scalar.weight\n",
      "net2.scalar.bias\n",
      "net2.magnetic.weight\n",
      "net2.magnetic.bias\n",
      "net2.dipole.weight\n",
      "net2.dipole.bias\n",
      "net2.potential.weight\n",
      "net2.potential.bias\n",
      "net2.type_embedding.weight\n",
      "net2.atom_embedding.weight\n"
     ]
    }
   ],
   "source": [
    "#for p in learner.model.parameters():\n",
    "#    if p.dim() == 1:\n",
    "#        print(p)\n",
    "        \n",
    "for name, param in learner.model.named_parameters():\n",
    "    if True: # or param.requires_grad and param.dim() == 1:\n",
    "        print(name)\n",
    "        #print(name, param.data)\n",
    "        #if len(name)  > 3:\n",
    "        #    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:09.065390Z",
     "start_time": "2019-07-26T08:58:58.751017Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-2.9936874, tensor(-3.0325, device='cuda:0'), tensor(-1.9991, device='cuda:0'), tensor(-3.5640, device='cuda:0'), tensor(-2.1114, device='cuda:0'), tensor(-3.5457, device='cuda:0'), tensor(-2.9502, device='cuda:0'), tensor(-3.5708, device='cuda:0'), tensor(-2.8017, device='cuda:0'), tensor(-3.7169, device='cuda:0')]\n"
     ]
    }
   ],
   "source": [
    "learner.to_fp32()\n",
    "\n",
    "val = learner.validate(dl=data.train_dl)\n",
    "\n",
    "print(val)\n",
    "val = val[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:20.611763Z",
     "start_time": "2019-07-26T09:01:19.884495Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss-2.9906val-3.0325\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    sub_fname = f'loss{learner.recorder.losses[-1]:.04f}val{val:.04f}'\n",
    "except:\n",
    "    sub_fname = f'val{val:.04f}'\n",
    "learner.save(sub_fname)\n",
    "print(sub_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T04:18:54.904735Z",
     "start_time": "2019-07-26T04:18:54.896522Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'loss-4.2748'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_fname = f'loss{learner.recorder.losses[-1]:.04f}'\n",
    "sub_fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-25T19:50:09.598169Z",
     "start_time": "2019-07-25T19:50:08.637758Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 7904968\r\n",
      "-rw-r--r-- 1 root root   50566405 Jul  8 07:27 loss-3.9978val-1.9089.pth\r\n",
      "-rw-r--r-- 1 root root   50985264 Jul  8 07:27 loss-3.9395val-1.9417.pth\r\n",
      "-rw-r--r-- 1 root root   50495731 Jul  8 07:27 loss-2.7748val-1.9216.pth\r\n",
      "-rw-r--r-- 1 root root   50500378 Jul  8 07:27 loss-4.4237val-1.9855.pth\r\n",
      "-rw-r--r-- 1 root root   50500214 Jul  8 07:27 loss-4.1809val-1.9292.pth\r\n",
      "-rw-r--r-- 1 root root   50566405 Jul  8 07:27 loss-3.8007val-1.8406.pth\r\n",
      "-rw-r--r-- 1 root root   42477068 Jul  8 07:27 loss-2.7250val-1.8349.pth\r\n",
      "-rw-r--r-- 1 root root   42477217 Jul  8 07:27 loss-2.8093val-1.8602.pth\r\n",
      "-rw-r--r-- 1 root root   50567976 Jul  8 07:27 loss-3.7100val-1.9721.pth\r\n",
      "-rw-r--r-- 1 root root   42421826 Jul  8 07:27 val-1.7450.pth\r\n",
      "-rw-r--r-- 1 root root   50987950 Jul  8 16:42 loss-3.7110val-1.9794.pth\r\n",
      "-rw-r--r-- 1 root root   51369339 Jul  9 07:00 loss-3.6805val-1.9788.pth\r\n",
      "-rw-r--r-- 1 root root   54983710 Jul 10 09:43 loss-3.8031val-1.9859.pth\r\n",
      "-rw-r--r-- 1 root root   54983942 Jul 10 09:43 loss-3.8269val-1.9933.pth\r\n",
      "-rw-r--r-- 1 root root   54986068 Jul 10 17:54 loss-3.7727val-1.9890.pth\r\n",
      "-rw-r--r-- 1 root root   54985826 Jul 12 13:07 loss-3.6655val-1.9898.pth\r\n",
      "-rw-r--r-- 1 root root   80199686 Jul 13 10:20 loss-3.9585val-2.0609.pth\r\n",
      "-rw-r--r-- 1 root root   80199266 Jul 13 15:37 loss0.1779val-2.0310.pth\r\n",
      "-rw------- 1 root root        137 Jul 13 15:40 tmpzs94ikyy\r\n",
      "-rw------- 1 root root        137 Jul 13 15:41 tmpm4w4vftj\r\n",
      "-rw-r--r-- 1 root root   80199266 Jul 13 19:46 loss0.2609val-2.0590.pth\r\n",
      "-rw-r--r-- 1 root root   80199266 Jul 14 06:32 loss0.5334val-1.1000.pth\r\n",
      "-rw-r--r-- 1 root root   80199266 Jul 14 08:41 loss0.1849val-2.5600.pth\r\n",
      "-rw-r--r-- 1 root root  227855040 Jul 15 00:53 loss-3.4306val-2.5506.pth\r\n",
      "-rw-r--r-- 1 root root   42951842 Jul 15 01:03 loss-3.3996val-2.4482.pth\r\n",
      "-rw-r--r-- 1 root root  227855204 Jul 16 02:01 loss-4.0414val-2.7287.pth\r\n",
      "-rw-r--r-- 1 root root   42951744 Jul 16 06:15 loss-3.1727val-2.4738.pth\r\n",
      "-rw-r--r-- 1 root root   42951678 Jul 16 06:15 loss-3.4640val-2.4699.pth\r\n",
      "-rw-r--r-- 1 root root  227855040 Jul 16 06:16 loss-4.0708val-2.6086.pth\r\n",
      "-rw-r--r-- 1 root root  908650352 Jul 18 06:22 loss-3.6971val-2.7181.pth\r\n",
      "-rw-r--r-- 1 root root   42780159 Jul 18 11:12 loss-3.6415val-2.5139.pth\r\n",
      "-rw-r--r-- 1 root root  984241202 Jul 23 06:11 loss-5.9943val-2.7766.pth_bak\r\n",
      "-rw-r--r-- 1 root root  984241202 Jul 23 06:12 loss-5.9943val-2.7766.pth\r\n",
      "-rw-r--r-- 1 root root  439696940 Jul 24 17:47 loss-3.1921val-2.3231.pth\r\n",
      "-rw-r--r-- 1 root root   73380929 Jul 25 15:29 tmp.pth\r\n",
      "-rw-r--r-- 1 root root  220124230 Jul 25 15:49 bestmodel.pth\r\n",
      "-rw-r--r-- 1 root root 1212098773 Jul 25 19:12 loss-4.5879val-2.8896.pth\r\n",
      "drwxr-xr-x 7 root root       8192 Jul 25 19:49 ..\r\n",
      "drwxr-xr-x 2 root root       4096 Jul 25 19:49 .\r\n",
      "-rw-r--r-- 1 root root 1212098776 Jul 25 19:49 loss-4.6372val-2.8869.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls -altr models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference\n",
    "\n",
    "Make sure `tranforms` are activated to test set otherwise TTA > 1 will be as TTA =1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:31.972918Z",
     "start_time": "2019-07-26T09:01:31.250345Z"
    }
   },
   "outputs": [],
   "source": [
    "test_fname = Path('test.npz')\n",
    "try:\n",
    "    npzfile  = np.load(fname_ext(test_fname, ext))\n",
    "    xt_xyz   = npzfile['x_xyz']\n",
    "    xt_type  = npzfile['x_type']\n",
    "    xt_ext   = npzfile['x_ext']\n",
    "    xt_atom  = npzfile['x_atom']\n",
    "    mt = npzfile['m']\n",
    "    xt_ids = npzfile['x_ids']\n",
    "except:\n",
    "    xt_xyz,xt_type,xt_ext,xt_atom,mt,xt_ids = \\\n",
    "        preprocess(test_fname.with_suffix('.csv'), type_index=types,ext=ext)\n",
    "    np.savez(fname_ext('_'+test_fname, ext), \n",
    "             x_xyz  = xt_xyz,\n",
    "             x_type = xt_type,\n",
    "             x_ext  = xt_ext,\n",
    "             x_atom = xt_atom,\n",
    "             m=mt,\n",
    "             x_ids=xt_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:34.619349Z",
     "start_time": "2019-07-26T09:01:34.609662Z"
    }
   },
   "outputs": [],
   "source": [
    "if True:\n",
    "    try:\n",
    "        xt_coulombmat = load_fn(f'xt_coulombmat32{ext}.npy')\n",
    "    except:\n",
    "        xt_coulombmat = np.load(f'xt_coulombmat{ext}.npy', allow_pickle=True)\n",
    "        xt_coulombmat = np.array(xt_coulombmat.tolist()).astype(np.float32)\n",
    "        np.save(f'xt_coulombmat32{ext}.npy', xt_coulombmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:35.895907Z",
     "start_time": "2019-07-26T09:01:35.889081Z"
    }
   },
   "outputs": [],
   "source": [
    "xt_qm9_mulliken = load_fn(f'xt_qm9_mulliken{ext}.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:37.869268Z",
     "start_time": "2019-07-26T09:01:37.862005Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'xt_qm9_mulliken_ext.npy'"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f'xt_qm9_mulliken{ext}.npy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:01:38.710603Z",
     "start_time": "2019-07-26T09:01:38.700027Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(756113, 3, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 1, 29),\n",
       " (756113, 29),\n",
       " (756113,)]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.shape for v in [xt_xyz,xt_type,xt_ext,xt_atom, xt_qm9_mulliken,xt_ids, mt]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:02:27.517101Z",
     "start_time": "2019-07-26T09:01:41.050050Z"
    }
   },
   "outputs": [],
   "source": [
    "TTA_N = 1\n",
    "\n",
    "learner.data.add_test(ItemList(items=(MoleculeItem(i,*v) for i,v in \n",
    "                              enumerate(zip(xt_xyz,xt_type,xt_ext,xt_atom,xt_qm9_mulliken,xt_coulombmat)))))\n",
    "learner.data.test_ds.tfms = tta_tfms if TTA_N > 1 else tfms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-18T06:25:49.739617Z",
     "start_time": "2019-07-18T06:25:49.719930Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'xt_xyz' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-59-749e57c7c67a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mgc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mdel\u001b[0m \u001b[0mxt_xyz\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxt_type\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxt_ext\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxt_atom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mgc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'xt_xyz' is not defined"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "del xt_xyz,xt_type,xt_ext,xt_atom, xt_qm9_mulliken, mt\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-16T06:05:22.050993Z",
     "start_time": "2019-07-16T06:05:22.048033Z"
    }
   },
   "outputs": [],
   "source": [
    "learner.model = nn.DataParallel(learner.model)\n",
    "#data.batch_size = int(4096*2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:14:31.778276Z",
     "start_time": "2019-07-26T09:02:27.519488Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>scalar_coupling_constant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4658147</td>\n",
       "      <td>11.239086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4658148</td>\n",
       "      <td>175.132706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4658149</td>\n",
       "      <td>5.385847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4658150</td>\n",
       "      <td>177.366669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4658151</td>\n",
       "      <td>12.400422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  scalar_coupling_constant\n",
       "0  4658147                 11.239086\n",
       "1  4658148                175.132706\n",
       "2  4658149                  5.385847\n",
       "3  4658150                177.366669\n",
       "4  4658151                 12.400422"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = defaultdict(int)\n",
    "xt_ids_not_extended = (xt_ids!=0) & (xt_ids<=7163688) # TODO\n",
    "ids = xt_ids[xt_ids_not_extended]\n",
    "\n",
    "mb = master_bar(range(TTA_N))\n",
    "for tta in mb:\n",
    "    test_preds = np.zeros((0, 29), dtype=np.float32)\n",
    "\n",
    "    for batch_idx, batch in progress_bar(\n",
    "        enumerate(learner.dl(DatasetType.Test)), total=len(learner.dl(DatasetType.Test)), parent=mb):\n",
    "        _, _, preds_,_,_,_ = learner.pred_batch(ds_type=DatasetType.Test, batch=batch)\n",
    "        preds_ = preds_.sum(dim=1)\n",
    "        test_preds = np.concatenate([test_preds, preds_.data.cpu().numpy()], axis = 0)\n",
    "\n",
    "    preds = test_preds[xt_ids_not_extended]\n",
    "    for k in range(len(ids)):\n",
    "        sub[int(ids[k])] += preds[k]\n",
    "    \n",
    "for k in range(len(ids)):\n",
    "    sub[int(ids[k])] = sub[int(ids[k])]/TTA_N\n",
    "\n",
    "sub_df = pd.DataFrame(sub.items(), columns=['id', 'scalar_coupling_constant'])\n",
    "sub_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submit to Kaggle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:14:31.784670Z",
     "start_time": "2019-07-26T09:14:31.780938Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'loss-2.9906val-3.0325'"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_fname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:14:43.311412Z",
     "start_time": "2019-07-26T09:14:31.786681Z"
    }
   },
   "outputs": [],
   "source": [
    "sub_df.to_csv(sub_fname, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:14:43.316470Z",
     "start_time": "2019-07-26T09:14:43.313662Z"
    }
   },
   "outputs": [],
   "source": [
    "comp = 'champs-scalar-coupling'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:14:43.326582Z",
     "start_time": "2019-07-26T09:14:43.318313Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>scalar_coupling_constant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2505537</th>\n",
       "      <td>7163684</td>\n",
       "      <td>0.760553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2505538</th>\n",
       "      <td>7163685</td>\n",
       "      <td>4.325841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2505539</th>\n",
       "      <td>7163686</td>\n",
       "      <td>1.800514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2505540</th>\n",
       "      <td>7163687</td>\n",
       "      <td>4.322571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2505541</th>\n",
       "      <td>7163688</td>\n",
       "      <td>119.676559</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              id  scalar_coupling_constant\n",
       "2505537  7163684                  0.760553\n",
       "2505538  7163685                  4.325841\n",
       "2505539  7163686                  1.800514\n",
       "2505540  7163687                  4.322571\n",
       "2505541  7163688                119.676559"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-16T06:11:03.974501Z",
     "start_time": "2019-07-16T06:11:03.969711Z"
    }
   },
   "outputs": [],
   "source": [
    "TTA_N=1\n",
    "ext='_ext'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T09:14:54.570811Z",
     "start_time": "2019-07-26T09:14:49.292288Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████| 64.0M/64.0M [00:02<00:00, 27.5MB/s]\n",
      "Successfully submitted to Predicting Molecular Properties"
     ]
    }
   ],
   "source": [
    "!kaggle competitions submit -c {comp} -f {sub_fname} -m ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T04:37:03.989952Z",
     "start_time": "2019-07-26T04:36:02.089675Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "time.sleep(60)\n",
    "!kaggle competitions submissions -c {comp} -v > submissions-{comp}.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-26T04:37:04.011120Z",
     "start_time": "2019-07-26T04:37:03.995305Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'-2.850'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submissions = pd.read_csv(f'submissions-{comp}.csv')\n",
    "submissions.iloc[0].publicScore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}