inception cifar 100 semisup large batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import logging\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "from common import data, fit\n",
    "from common.util import download_file\n",
    "import mxnet as mx\n",
    "import numpy as np\n",
    "from mxnet.symbol import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_cifar100():\n",
    "    data_dir=\"/efs/data/cifar-100-mxnet\"\n",
    "    #data_dir=\"/data/cifar-100-mxnet\"\n",
    "    fnames = (os.path.join(data_dir, \"train.rec\"),\n",
    "              os.path.join(data_dir, \"test.rec\"))\n",
    "    return fnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(train_fname, val_fname) = download_cifar100()\n",
    "\n",
    "# parse args\n",
    "parser = argparse.ArgumentParser(description=\"train cifar100\",\n",
    "                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)\n",
    "fit.add_fit_args(parser)\n",
    "data.add_data_args(parser)\n",
    "data.add_data_aug_args(parser)\n",
    "data.set_data_aug_level(parser, 2)\n",
    "\n",
    "num_unsup_examples = 50000\n",
    "subset_factor = 5\n",
    "\n",
    "parser.set_defaults(\n",
    "    # data\n",
    "    data_train     = train_fname,\n",
    "    data_val       = val_fname,\n",
    "    num_classes    = 100,\n",
    "    image_shape    = '3,28,28',\n",
    "    prefix         = 'cifar_b250_fine',\n",
    "    pad_size       = 2,\n",
    "    # train\n",
    "    batch_size     = 250,  # todo currently has to devide 'num_validation_samples'\n",
    "    num_epochs     = 380,\n",
    "    lr_step_epochs = '180,270',  # this setting should converge to a good result\n",
    "    gpus           = '0,1,2,3,4,5,6,7',\n",
    "    loss_gpu       = 0,\n",
    "    disp_batches   = 20,\n",
    "    num_examples   = num_unsup_examples / subset_factor, \n",
    "    wd             = 1e-4,\n",
    "    lr             = .001,\n",
    "    lr_factor      = .33,\n",
    "    nembeddings    = 256,\n",
    "    optimizer      = 'adam'\n",
    ")\n",
    "\n",
    "args = parser.parse_args(\"\")\n",
    "\n",
    "unsup_multiplier = 2\n",
    "labeled_per_class = 500 / subset_factor\n",
    "\n",
    "sample_seed = 47\n",
    "val_interval = 1\n",
    "save_interval = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4  times more unsup data than sup data\n"
     ]
    }
   ],
   "source": [
    "from common.multi_iterator import Multi_iterator\n",
    "from common.data import get_partial_rec_iter\n",
    "\n",
    "(train_sup, val) = get_partial_rec_iter(args, get_val=True, devide_by=subset_factor, shuffle=True)\n",
    "(train_unsup, _) = get_partial_rec_iter(args, get_val=False, devide_by=1, shuffle=True)\n",
    "    \n",
    "num_sup_examples = labeled_per_class * args.num_classes\n",
    "train = Multi_iterator(train_sup, train_unsup, unsup_multiplier, num_unsup_examples, num_sup_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3600.0]\n"
     ]
    }
   ],
   "source": [
    "from symbols import inception_cifar as base_net\n",
    "from common.lba import compute_semisup_loss, logit_loss\n",
    "from common.wrapper_module import get_embedding_shapes\n",
    "\n",
    "# kvstore\n",
    "kv = mx.kvstore.create(args.kv_store)\n",
    "\n",
    "t_nb = args.batch_size * unsup_multiplier\n",
    "#initializer = mx.init.Xavier(rnd_type='gaussian', factor_type=\"avg\", magnitude=2.34)\n",
    "initializer = mx.init.Uniform(0.01)\n",
    "\n",
    "lr, lr_scheduler = fit._get_lr_scheduler(args, kv)\n",
    "\n",
    "def buildEmbeddingModule(arg_p=None, aux_p=None):\n",
    "    dataSup = mx.symbol.Variable(name=\"dataSup\")\n",
    "    dataUnsup = []\n",
    "    for i in range(unsup_multiplier):\n",
    "        dataUnsup.append(Variable(name=\"dataUnsup\"+str(i)))\n",
    "\n",
    "    # concat data, feed both through the network\n",
    "    # then split it up again\n",
    "    data = concat(dataSup, *dataUnsup, dim=0)\n",
    "\n",
    "    embeddings = base_net.build_embeddings(data, nembeddings=args.nembeddings)\n",
    "    splitted = split(embeddings, num_outputs=(unsup_multiplier+1), axis=0, name='split')\n",
    "    \n",
    "    # devices for training\n",
    "    devs = mx.cpu() if args.gpus is None or args.gpus is '' else [\n",
    "        mx.gpu(int(i)) for i in args.gpus.split(',')]\n",
    "\n",
    "    data_names = ['dataSup'] + ['dataUnsup'+str(i) for i in range(unsup_multiplier)]\n",
    "        \n",
    "    # create model\n",
    "    model = mx.mod.Module(\n",
    "        context       = devs,\n",
    "        symbol        = splitted,\n",
    "        data_names    = data_names,\n",
    "        label_names   = None)\n",
    "    \n",
    "    model.bind(data_shapes=train.provide_data)         \n",
    "    model.init_params(initializer, arg_p, aux_p)    \n",
    "    model.init_optimizer(optimizer=args.optimizer, optimizer_params=(\n",
    "       ('learning_rate', lr), \n",
    "       ('wd', 1e-4),\n",
    "       #('momentum', args.mom),\n",
    "       ('rescale_grad', 0.005),\n",
    "       ('lr_scheduler', lr_scheduler)))\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "def buildSupLossModule(arg_p=None, aux_p=None):\n",
    "    supEmbeddings = Variable(name=\"embeddings_sup\")\n",
    "    labelSup = mx.symbol.Variable(name='labelSup')\n",
    "    overall_loss = []\n",
    "    \n",
    "    if unsup_multiplier >= 1:\n",
    "        unsupEmbeddings = []\n",
    "        for i in range(unsup_multiplier):\n",
    "            unsupEmbeddings.append(Variable(name=\"embeddings_unsup\"+str(i)))\n",
    "    \n",
    "        unsupEmbeddings = concat(*unsupEmbeddings, dim=0)\n",
    "            \n",
    "        (walker_loss, visit_loss) = compute_semisup_loss(supEmbeddings, unsupEmbeddings, labelSup, t_nb, \n",
    "                                                     walker_weight=1.0, visit_weight=0.5)\n",
    "        #overall_loss = [walker_loss, visit_loss]\n",
    "        \n",
    "    overall_loss = [logit_loss(supEmbeddings, labelSup, args.num_classes)] + overall_loss\n",
    "    \n",
    "    # todo maybe use gpu\n",
    "    devs = mx.cpu() if args.loss_gpu is None or args.loss_gpu is '' else mx.gpu(args.loss_gpu)\n",
    "\n",
    "    # create module\n",
    "    model = mx.mod.Module(\n",
    "        context = devs,\n",
    "        symbol  = Group(overall_loss),\n",
    "        data_names = ['embeddings_sup'],#+['embeddings_unsup'+str(i) for i in range(unsup_multiplier)],\n",
    "        label_names = ['labelSup'])\n",
    "    \n",
    "    # allocate memory by given the input data and label shapes\n",
    "    model.bind(data_shapes=[get_embedding_shapes(args.batch_size, args.nembeddings, unsup_multiplier)[0]], \n",
    "               label_shapes=train.provide_label,\n",
    "               inputs_need_grad=True)\n",
    "            \n",
    "    model.init_params(initializer, arg_p, aux_p)    \n",
    "    model.init_optimizer(optimizer=args.optimizer, optimizer_params=(\n",
    "        ('learning_rate', lr), \n",
    "        ('rescale_grad', 0.005),\n",
    "        #('momentum', args.mom),\n",
    "        ('wd', 1e-4),\n",
    "        ('lr_scheduler', lr_scheduler)))\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "def buildLossModule(arg_p=None, aux_p=None):\n",
    "    supEmbeddings = Variable(name=\"embeddings_sup\")\n",
    "    labelSup = mx.symbol.Variable(name='labelSup')\n",
    "    overall_loss = []\n",
    "    \n",
    "    if unsup_multiplier >= 1:\n",
    "        unsupEmbeddings = []\n",
    "        for i in range(unsup_multiplier):\n",
    "            unsupEmbeddings.append(Variable(name=\"embeddings_unsup\"+str(i)))\n",
    "    \n",
    "        unsupEmbeddings = concat(*unsupEmbeddings, dim=0)\n",
    "            \n",
    "        (walker_loss, visit_loss) = compute_semisup_loss(supEmbeddings, unsupEmbeddings, labelSup, t_nb, \n",
    "                                                     walker_weight=0.8, visit_weight=0.5)\n",
    "        overall_loss = [walker_loss, visit_loss]\n",
    "        \n",
    "    overall_loss = [logit_loss(supEmbeddings, labelSup, args.num_classes)] + overall_loss\n",
    "    \n",
    "    # todo maybe use gpu\n",
    "    devs = mx.cpu() if args.loss_gpu is None or args.loss_gpu is '' else mx.gpu(args.loss_gpu)\n",
    "\n",
    "    # create module\n",
    "    model = mx.mod.Module(\n",
    "        context = devs,\n",
    "        symbol  = Group(overall_loss),\n",
    "        data_names = ['embeddings_sup']+['embeddings_unsup'+str(i) for i in range(unsup_multiplier)],\n",
    "        label_names = ['labelSup'])\n",
    "    \n",
    "    # allocate memory by given the input data and label shapes\n",
    "    model.bind(data_shapes=get_embedding_shapes(args.batch_size, args.nembeddings, unsup_multiplier), \n",
    "               label_shapes=train.provide_label,\n",
    "               inputs_need_grad=True)\n",
    "            \n",
    "    model.init_params(initializer, arg_p, aux_p)    \n",
    "    model.init_optimizer(optimizer=args.optimizer, optimizer_params=(\n",
    "        ('learning_rate', lr), \n",
    "        ('rescale_grad', 0.005),\n",
    "        #('momentum', args.mom),\n",
    "        ('wd', 1e-4),\n",
    "        ('lr_scheduler', lr_scheduler)))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from common.wrapper_module import WrapperModule\n",
    "#eval_metrics = Multi_Accuracy(num= 3 if unsup_multiplier >= 1 else 1)\n",
    "                    \n",
    "def fit_model(args, embeddingModule, lossModule, data, **kwargs):\n",
    "    \"\"\"\n",
    "    train a model\n",
    "    args : argparse returns\n",
    "    data_loader : function that returns the train and val data iterators\n",
    "    \"\"\"\n",
    "    # logging\n",
    "    head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s'\n",
    "    logging.basicConfig(level=logging.DEBUG, format=head)\n",
    "    logging.info('start with arguments %s', args)\n",
    "    \n",
    "    # data iterators\n",
    "    (train, val) = data\n",
    "\n",
    "    # load model\n",
    "    if 'arg_params' in kwargs and 'aux_params' in kwargs:\n",
    "        arg_params = kwargs['arg_params']\n",
    "        aux_params = kwargs['aux_params']\n",
    "    else: \n",
    "        sym, arg_params, aux_params = fit._load_model(args, kv.rank)\n",
    "        \n",
    "    # save model\n",
    "    checkpoint = fit._save_model(args, kv.rank)\n",
    "\n",
    "    model = WrapperModule(embeddingModule, lossModule, unsup_multiplier)\n",
    "    \n",
    "    logf = open(args.prefix+'logs', 'w')\n",
    "    def validate_model(epoch, *args_):\n",
    "        if epoch % val_interval != 0: \n",
    "            return\n",
    "        res = model.score(val)\n",
    "        #TODO: pull this into default\n",
    "        print('Epoch[%d] Validation-accuracy=%f' % (epoch,  res))\n",
    "        logf.write('Epoch[%d] Validation-accuracy=%f \\n' % (epoch,  res))  # python will convert \\n to os.linesep\n",
    "        logf.flush()\n",
    "\n",
    "        if epoch % save_interval == 0:\n",
    "            model.save_checkpoint(args.prefix, epoch)\n",
    "\n",
    "    #monitor = mx.mon.Monitor(interval=1000, pattern='.*aba_backward.*') \n",
    "    monitor = mx.mon.Monitor(interval=1000, pattern='.*') \n",
    "    \n",
    "    # callbacks that run after each batch\n",
    "    batch_end_callbacks = [mx.callback.Speedometer(args.batch_size*(unsup_multiplier+1), args.disp_batches)]\n",
    "    \n",
    "    epoch_end_callbacks = validate_model\n",
    "\n",
    "    # run\n",
    "    model.fit(train,\n",
    "        begin_epoch        = args.load_epoch if args.load_epoch else 0,\n",
    "        num_epoch          = args.num_epochs,\n",
    "        #eval_data          = val,\n",
    "        #eval_metric        = eval_metrics,\n",
    "        kvstore            = kv,\n",
    "        arg_params         = arg_params,\n",
    "        aux_params         = aux_params,\n",
    "        batch_end_callback = batch_end_callbacks,\n",
    "        epoch_end_callback = epoch_end_callbacks,\n",
    "        allow_missing      = True\n",
    "        #monitor            = monitor\n",
    "             )\n",
    "    logf.close()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:start with arguments Namespace(batch_size=250, benchmark=0, data_nthreads=4, data_train='/efs/data/cifar-100-mxnet/train.rec', data_val='/efs/data/cifar-100-mxnet/test.rec', disp_batches=20, dtype='float32', gpus='0,1,2,3,4,5,6,7', image_shape='3,28,28', kv_store='device', load_epoch=None, loss_gpu=0, lr=0.00033, lr_factor=0.33, lr_step_epochs='90', max_random_aspect_ratio=0, max_random_h=36, max_random_l=50, max_random_rotate_angle=0, max_random_s=50, max_random_scale=1, max_random_shear_ratio=0, min_random_scale=1, model_prefix=None, mom=0.9, monitor=0, nembeddings=256, network=None, num_classes=100, num_epochs=200, num_examples=10000.0, num_layers=None, optimizer='adam', pad_size=2, prefix='cifar_b250_fine', random_crop=1, random_mirror=1, rgb_mean='123.68,116.779,103.939', test_io=0, top_k=0, wd=0.0001)\n",
      "INFO:root:Epoch[0] Batch [20]\tSpeed: 884.80 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[0] Train-accuracy=nan\n",
      "INFO:root:Epoch[0] Time cost=35.060\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[0] Validation-accuracy=0.506700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Saved checkpoint to \"emb_cifar_b250_fine-0000.params\"\n",
      "INFO:root:Saved checkpoint to \"loss_cifar_b250_fine-0000.params\"\n",
      "INFO:root:Epoch[1] Batch [20]\tSpeed: 865.74 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[1] Train-accuracy=nan\n",
      "INFO:root:Epoch[1] Time cost=34.209\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[1] Validation-accuracy=0.507900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[2] Batch [20]\tSpeed: 867.51 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[2] Train-accuracy=nan\n",
      "INFO:root:Epoch[2] Time cost=34.360\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[2] Validation-accuracy=0.508700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[3] Batch [20]\tSpeed: 862.55 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[3] Train-accuracy=nan\n",
      "INFO:root:Epoch[3] Time cost=34.319\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[3] Validation-accuracy=0.505500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[4] Batch [20]\tSpeed: 867.50 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[4] Train-accuracy=nan\n",
      "INFO:root:Epoch[4] Time cost=34.273\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[4] Validation-accuracy=0.500200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[5] Batch [20]\tSpeed: 862.36 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[5] Train-accuracy=nan\n",
      "INFO:root:Epoch[5] Time cost=34.505\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[5] Validation-accuracy=0.505200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[6] Batch [20]\tSpeed: 862.19 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[6] Train-accuracy=nan\n",
      "INFO:root:Epoch[6] Time cost=34.446\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[6] Validation-accuracy=0.507800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[7] Batch [20]\tSpeed: 859.65 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[7] Train-accuracy=nan\n",
      "INFO:root:Epoch[7] Time cost=34.425\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[7] Validation-accuracy=0.507800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[8] Batch [20]\tSpeed: 864.50 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[8] Train-accuracy=nan\n",
      "INFO:root:Epoch[8] Time cost=34.437\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[8] Validation-accuracy=0.509500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[9] Batch [20]\tSpeed: 865.82 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[9] Train-accuracy=nan\n",
      "INFO:root:Epoch[9] Time cost=34.442\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[9] Validation-accuracy=0.507700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[10] Batch [20]\tSpeed: 865.14 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[10] Train-accuracy=nan\n",
      "INFO:root:Epoch[10] Time cost=34.373\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[10] Validation-accuracy=0.508600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Saved checkpoint to \"emb_cifar_b250_fine-0010.params\"\n",
      "INFO:root:Saved checkpoint to \"loss_cifar_b250_fine-0010.params\"\n",
      "INFO:root:Epoch[11] Batch [20]\tSpeed: 865.75 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[11] Train-accuracy=nan\n",
      "INFO:root:Epoch[11] Time cost=34.374\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[11] Validation-accuracy=0.505600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[12] Batch [20]\tSpeed: 866.84 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[12] Train-accuracy=nan\n",
      "INFO:root:Epoch[12] Time cost=34.402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[12] Validation-accuracy=0.506600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[13] Batch [20]\tSpeed: 862.83 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[13] Train-accuracy=nan\n",
      "INFO:root:Epoch[13] Time cost=34.413\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[13] Validation-accuracy=0.510600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[14] Batch [20]\tSpeed: 864.81 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[14] Train-accuracy=nan\n",
      "INFO:root:Epoch[14] Time cost=34.363\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[14] Validation-accuracy=0.507600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[15] Batch [20]\tSpeed: 861.89 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[15] Train-accuracy=nan\n",
      "INFO:root:Epoch[15] Time cost=34.456\n",
      "INFO:root:Epoch[159] Batch [20]\tSpeed: 862.63 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[159] Train-accuracy=nan\n",
      "INFO:root:Epoch[159] Time cost=34.431\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[159] Validation-accuracy=0.544100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Epoch[160] Batch [20]\tSpeed: 863.57 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[160] Train-accuracy=nan\n",
      "INFO:root:Epoch[160] Time cost=34.387\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[160] Validation-accuracy=0.537300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:root:Saved checkpoint to \"emb_cifar_b250_fine-0160.params\"\n",
      "INFO:root:Saved checkpoint to \"loss_cifar_b250_fine-0160.params\"\n",
      "INFO:root:Epoch[161] Batch [20]\tSpeed: 865.29 samples/sec\tTrain-accuracy=nan\n",
      "INFO:root:Epoch[161] Train-accuracy=nan\n",
      "INFO:root:Epoch[161] Time cost=34.350\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch[161] Validation-accuracy=0.541300\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-9015f90711da>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m#lossModule = buildSupLossModule()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membeddingModule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlossModule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-9-2bf31653fa3d>\u001b[0m in \u001b[0;36mfit_model\u001b[0;34m(args, embeddingModule, lossModule, data, **kwargs)\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mbatch_end_callback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_end_callbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m         \u001b[0mepoch_end_callback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mepoch_end_callbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0mallow_missing\u001b[0m      \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m         \u001b[0;31m#monitor            = monitor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m              )\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/mxnet/module/base_module.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, train_data, eval_data, eval_metric, epoch_end_callback, batch_end_callback, kvstore, optimizer, optimizer_params, eval_end_callback, eval_batch_end_callback, initializer, arg_params, aux_params, allow_missing, force_rebind, force_init, begin_epoch, num_epoch, validation_metric, monitor)\u001b[0m\n\u001b[1;32m    470\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mmonitor\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m                     \u001b[0mmonitor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 472\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_backward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    473\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/mxnet/module/base_module.py\u001b[0m in \u001b[0;36mforward_backward\u001b[0;34m(self, data_batch)\u001b[0m\n\u001b[1;32m    192\u001b[0m         \u001b[0;34m\"\"\"A convenient function that calls both ``forward`` and ``backward``.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m     def score(self, eval_data, eval_metric, num_batch=None, batch_end_callback=None,\n",
      "\u001b[0;32m/efs/mxnet/example/image-classification/common/wrapper_module.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     44\u001b[0m         \u001b[0;31m# synchronize - otherwise results will be NaNs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mgrad\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgrads\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m             \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m          \u001b[0;31m#   print(\"grad\",a)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/mxnet/ndarray.py\u001b[0m in \u001b[0;36masnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    739\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    740\u001b[0m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata_as\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mctypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mc_void_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 741\u001b[0;31m             ctypes.c_size_t(data.size)))\n\u001b[0m\u001b[1;32m    742\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    743\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train.reset()\n",
    "\n",
    "# train using a checkpoint with 20% validation accuracy, trained only supervised\n",
    "#(sym, arg_p, aux_p) = mx.model.load_checkpoint('embedding_val30',25)\n",
    "#embeddingModule = buildEmbeddingModule(arg_p, aux_p)      \n",
    "\n",
    "# train from previous iteration\n",
    "(arg_p_emb, aux_p_emb, arg_p_loss, aux_p_loss) = WrapperModule.load_checkpoint('cifarsemisup', 200)\n",
    "embeddingModule = buildEmbeddingModule(arg_p_emb, aux_p_emb)      \n",
    "lossModule = buildLossModule(arg_p_loss, aux_p_loss)\n",
    "# train from scratch\n",
    "#embeddingModule = buildEmbeddingModule()\n",
    "\n",
    "#semisupervised loss module\n",
    "#lossModule = buildLossModule()\n",
    "\n",
    "#supervised loss module\n",
    "#lossModule = buildSupLossModule()\n",
    "\n",
    "m = fit_model(args, embeddingModule, lossModule, (train, val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aux_p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeddingModule.save_checkpoint('cifarsemisup200', 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lossModule.save_checkpoint('cifarsemisup200_loss', 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val.reset()\n",
    "val.next().label[0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Multi_Accuracy(mx.metric.EvalMetric):\n",
    "    \"\"\"Calculate accuracies of multi label\"\"\"\n",
    "\n",
    "    def __init__(self, num=None):\n",
    "        super(Multi_Accuracy, self).__init__('multi-accuracy', num)\n",
    "\n",
    "    def update(self, labels, preds):\n",
    "\n",
    "        #for i in range(len(preds)):\n",
    "        for i in range(1):\n",
    "            pred_label = mx.nd.argmax_channel(preds[i]).asnumpy().astype('int32')\n",
    "            label = labels[0].asnumpy().astype('int32')\n",
    "\n",
    "            #mx.metric.check_label_shapes(label, pred_label)\n",
    "            \n",
    "            #print((pred_label.flat == label.flat).sum())\n",
    "            #print(len(pred_label.flat))\n",
    "\n",
    "            \n",
    "            self.sum_metric[i] += (pred_label.flat == label.flat).sum()\n",
    "            self.num_inst[i] += len(pred_label.flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# batch 10 train 0.389, valid: 0.40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}