DCLU_full_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "55b06830",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gzip\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.python.platform import gfile\n",
    "from sklearn.preprocessing import normalize\n",
    "import os\n",
    "import tensorflow.compat.v1 as tf\n",
    "tf.disable_eager_execution()\n",
    "import logging\n",
    "from __future__ import print_function, division, absolute_import, unicode_literals\n",
    "from typing import Union, Any\n",
    "import shutil\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import scipy.ndimage as nd\n",
    "\n",
    "%matplotlib inline\n",
    "import pylab as pl\n",
    "from IPython import display\n",
    "from keras import backend as K_1, regularizers\n",
    "from keras.engine.training import Model\n",
    "from keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, \\\n",
    "    BatchNormalization, Activation, Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b5a60b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataSet(object):\n",
    "    def __init__(self, images, labels, probs, indices, reshape=True):\n",
    "        \n",
    "        if reshape:\n",
    "            mean = np.mean(images,axis=(0,1,2,3))\n",
    "            std =  np.std(images,axis=(0,1,2,3))\n",
    "            images = (images - mean)/std\n",
    "        \n",
    "        self._images = images\n",
    "        self._num_examples = images.shape[0]\n",
    "        self._labels = labels\n",
    "        self._probs = probs\n",
    "        self._epochs_completed = 0\n",
    "        self._index_in_epoch = 0\n",
    "        self._counter = np.zeros(self._num_examples)\n",
    "        self._indices = indices\n",
    "        self._subset_size = images.shape[0]\n",
    "        self._subset_ids = np.arange(self._num_examples)\n",
    "        self._data_limit = None\n",
    "        self._train_order = np.arange(self._num_examples)\n",
    "        self._epoch_ids = None\n",
    "        self._precent = 0.25\n",
    "        self._cur_precent = 0.25\n",
    "\n",
    "    @property\n",
    "    def images(self):\n",
    "        return self._images\n",
    "\n",
    "    @property\n",
    "    def labels(self):\n",
    "        return self._labels\n",
    "\n",
    "    @property\n",
    "    def probs(self):\n",
    "        return self._probs\n",
    "\n",
    "    @property\n",
    "    def indices(self):\n",
    "        return self._indices\n",
    "\n",
    "    @property\n",
    "    def num_examples(self):\n",
    "        return self._num_examples\n",
    "\n",
    "    @property\n",
    "    def epochs_completed(self):\n",
    "        return self._epochs_completed\n",
    "\n",
    "    @property\n",
    "    def counter(self):\n",
    "        return self._counter\n",
    "    \n",
    "    @property\n",
    "    def subset_size(self):\n",
    "        return self._subset_size\n",
    "\n",
    "    @property\n",
    "    def subset_ids(self):\n",
    "        return self._subset_ids\n",
    "    \n",
    "    def train_order(self):\n",
    "        pp = np.array(self._probs)\n",
    "        self._train_order = np.argsort(pp)\n",
    "        return self._train_order\n",
    "    \n",
    "    def revert_indicte(self):\n",
    "        self._indices = np.zeros(self._num_examples)\n",
    "    \n",
    "    def subset_step_indicate(self, epoch, increase_amount):\n",
    "        if epoch == 0:\n",
    "            self._data_limit = int(np.ceil(self._num_examples * self._precent))\n",
    "            self._epoch_ids = self._train_order[:self._data_limit]\n",
    "            self._indices[self._epoch_ids]  =  np.ones(self._data_limit)\n",
    "        else:\n",
    "            self._precent = min(self._cur_precent * increase_amount,1)\n",
    "            if self._precent != self._cur_precent:\n",
    "                self._cur_precent = self._precent\n",
    "                self._data_limit = int(np.ceil(self._num_examples * self._precent))\n",
    "                self._epoch_ids = self._train_order[:self._data_limit]\n",
    "                self._indices[self._epoch_ids]  =  np.ones(self._data_limit)\n",
    "            else:\n",
    "                self._data_limit = int(np.ceil(self._num_examples * self._precent))\n",
    "                self._epoch_ids = self._train_order[:self._data_limit]\n",
    "                self._indices[self._epoch_ids]  =  np.ones(self._data_limit)\n",
    "                \n",
    "                \n",
    "    def next_batch_train(self, batch_size):\n",
    "        start = self._index_in_epoch\n",
    "        self._index_in_epoch += batch_size\n",
    "        if self._index_in_epoch > self._num_examples:\n",
    "            self._epochs_completed += 1\n",
    "            start = 0\n",
    "            self._index_in_epoch = batch_size\n",
    "            assert batch_size <= self._num_examples\n",
    "        end = self._index_in_epoch\n",
    "        return self._images[self._train_order][start:end], \\\n",
    "                self._labels[self._train_order][start:end], \\\n",
    "                self._probs[self._train_order][start:end], \\\n",
    "                self._indices[self._train_order][start:end]    \n",
    "    \n",
    "    def next_batch_test(self, batch_size):    \n",
    "        start = self._index_in_epoch\n",
    "        self._index_in_epoch += batch_size\n",
    "        if self._index_in_epoch > self._num_examples:\n",
    "            # Finished epoch\n",
    "            self._epochs_completed += 1\n",
    "\n",
    "            # Shuffle the data\n",
    "            np.random.seed(1)\n",
    "            perm = np.arange(self._num_examples)\n",
    "            np.random.shuffle(perm)\n",
    "            self._images = self._images[perm]\n",
    "            self._labels = self._labels[perm]\n",
    "            self._probs = self._probs[perm]\n",
    "            self._indices = self._indices[perm]\n",
    "\n",
    "            # Start next epoch\n",
    "            start = 0\n",
    "            self._index_in_epoch = batch_size\n",
    "            assert batch_size <= self._num_examples\n",
    "        end = self._index_in_epoch\n",
    "        return self._images[start:end], self._labels[start:end], self._probs[start:end], self._indices[start:end]\n",
    "\n",
    "    def change_probs(self, new_values):\n",
    "        self._probs[self._train_order] = new_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6afaaf50",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data_sets(datasetname, init_probs=[], one_hot=False):\n",
    "    \n",
    "    class DataSets(object):\n",
    "        pass\n",
    "    data_sets = DataSets()\n",
    "    \n",
    "    if datasetname == 'cifar10_keras':\n",
    "        (train_images,train_labels),(test_images,test_labels) = cifar10.load_data()\n",
    "    elif datasetname == 'mnist_keras':\n",
    "        (train_images,train_labels),(test_images,test_labels) = mnist.load_data()\n",
    "        num_rows_train = train_images.shape[1]\n",
    "        num_clos_train = train_images.shape[2]\n",
    "        num_rows_test = test_images.shape[1]\n",
    "        num_clos_test = test_images.shape[2]\n",
    "        train_images = train_images.reshape(train_images.shape[0],num_rows_train,num_clos_train,1)\n",
    "        test_images = test_images.reshape(test_images.shape[0],num_rows_train,num_clos_train,1)\n",
    "        if K_1.image_data_format()=='channels_last':\n",
    "            train_images = train_images.transpose(0,1,2,3)\n",
    "            test_images = test_images.transpose(0,1,2,3)\n",
    "    else:\n",
    "        raise NotImplementedError('dataset not supported')\n",
    "    \n",
    "    if one_hot:\n",
    "        train_labels = tf.keras.utils.to_categorical(train_labels, num_classes=10)\n",
    "        test_labels = tf.keras.utils.to_categorical(test_labels, num_classes=10)\n",
    "  \n",
    "    n_test = test_images.shape[0]\n",
    "    n_train = train_images.shape[0]\n",
    "    \n",
    "    if not init_probs:\n",
    "        print('RANDOM INIT PROBABILITIES')\n",
    "        np.random.seed(3)\n",
    "        probs = np.random.rand(n_train)\n",
    "    else:\n",
    "        init_probs = np.asarray(init_probs)\n",
    "        probs_class = np.asarray(1.0 * init_probs / np.sum(init_probs), np.float32)\n",
    "        dense_train_labels = np.argmax(train_labels, axis=1)\n",
    "        probs = np.zeros_like(dense_train_labels, np.float32)\n",
    "        for k in range(0, np.unique(dense_train_labels).max()+1):\n",
    "            i = np.where(dense_train_labels == k)[0]\n",
    "            probs[i] = probs_class[k]\n",
    "\n",
    "    train_probs = np.squeeze(probs)\n",
    "    test_probs = np.squeeze(normalize(np.expand_dims(np.ones(n_test, np.float32), 1), axis=0, norm='l1'))\n",
    "    \n",
    "    train_indices = np.zeros(n_train)\n",
    "   \n",
    "    test_indices = np.zeros(n_test)\n",
    "    \n",
    "    data_sets.train = DataSet(train_images, train_labels, train_probs, train_indices)\n",
    "    \n",
    "    data_sets.test = DataSet(test_images, test_labels, test_probs, test_indices)\n",
    "    \n",
    "    return data_sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "afff66bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RANDOM INIT PROBABILITIES\n"
     ]
    }
   ],
   "source": [
    "dataset_1 = read_data_sets('cifar10_keras', init_probs=[],one_hot=True)\n",
    "train_set = dataset_1.train\n",
    "test_set = dataset_1.test\n",
    "K = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d4c2a3e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def relu_evidence(logits):\n",
    "    return tf.nn.relu(logits)\n",
    "\n",
    "def exp_evidence(logits): \n",
    "    return tf.exp(tf.clip_by_value(logits/10,-10,10))\n",
    "\n",
    "def softplus_evidence(logits):\n",
    "    return tf.nn.softplus(logits)\n",
    "def softsign_evidence(logits):\n",
    "    return tf.nn.softsign(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3d9565e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def var(name, shape, init=None):\n",
    "    init = tf.truncated_normal_initializer(stddev=(1 / shape[0]) ** 0.5) if init is None else init\n",
    "    return tf.get_variable(name=name, shape=shape, dtype=tf.float32, initializer=init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dae035d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def DEAL_dense_layer(x, units, rate, activation, scope='DEAL_dense_0'):\n",
    "    with tf.variable_scope(scope):\n",
    "        W3 = var('W3', [x.get_shape()[1], 1000])\n",
    "        b3 = var('b3', [1000])\n",
    "        out3 = Activation(activation=activation)(tf.matmul(x, W3) + b3)\n",
    "        out3 = Dropout(rate=rate)(out3)\n",
    "\n",
    "        W4 = var('W4', [1000, units])\n",
    "        b4 = var('b4', [units])\n",
    "        logits = tf.matmul(out3, W4) + b4\n",
    "        return logits, W3, W4\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1eac622e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def network(x, units=10,logits2evidence=softplus_evidence,activation='elu', dropout_1_rate=0.25, dropout_2_rate=0.5,\n",
    "            reg_factor=50e-4, bias_reg_factor=None, batch_norm=False):\n",
    "    \n",
    "    l2_reg = regularizers.l2(reg_factor)  # K.variable(K.cast_to_floatx(reg_factor))\n",
    "    l2_bias_reg = None\n",
    "    if bias_reg_factor:\n",
    "        l2_bias_reg = regularizers.l2(bias_reg_factor)  # K.variable(K.cast_to_floatx(bias_reg_factor))\n",
    "\n",
    "    x = Conv2D(filters=32, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg,\n",
    "               bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = Conv2D(filters=32, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg,\n",
    "                   bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    x = Dropout(rate=dropout_1_rate)(x)\n",
    "\n",
    "    x = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg,\n",
    "                bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = Conv2D(filters=64, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg,\n",
    "                   bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    x = Dropout(rate=dropout_1_rate)(x)\n",
    "\n",
    "    x = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg,\n",
    "                   bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg,\n",
    "                   bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    x = Dropout(rate=dropout_1_rate)(x)\n",
    "\n",
    "    x = Conv2D(filters=256, kernel_size=(2, 2), padding='same', kernel_regularizer=l2_reg,\n",
    "                bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = Conv2D(filters=256, kernel_size=(2, 2), padding='same', kernel_regularizer=l2_reg,\n",
    "                   bias_regularizer=l2_bias_reg)(x)\n",
    "    if batch_norm:\n",
    "        x = BatchNormalization()(x)\n",
    "    x = Activation(activation=activation)(x)\n",
    "    x = MaxPooling2D(pool_size=(2, 2))(x)\n",
    "    x = Dropout(rate=dropout_1_rate)(x)\n",
    "\n",
    "    x = Flatten()(x)\n",
    "    x,W3,W4 = DEAL_dense_layer(x, units, rate=dropout_2_rate, activation=activation, scope='logit')\n",
    "#     x = Dense(units=128, kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(x)\n",
    "#     if batch_norm:\n",
    "#         x = BatchNormalization()(x)\n",
    "#     x = Activation(activation=activation)(x)\n",
    "\n",
    "#     x = Dropout(rate=dropout_2_rate)(x)\n",
    "#     x = Dense(units=units, kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(x)\n",
    "    evidence = logits2evidence(x)\n",
    "    return x, evidence, W3, W4\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9aec0559",
   "metadata": {},
   "outputs": [],
   "source": [
    "def KL(alpha, K):\n",
    "    beta = tf.constant(np.ones((1, K)), dtype=tf.float32)\n",
    "    S_alpha = tf.reduce_sum(alpha, axis=1, keepdims=True)\n",
    "\n",
    "    KL = tf.reduce_sum((alpha - beta) * (tf.digamma(alpha) - tf.digamma(S_alpha)), axis=1, keepdims=True) + \\\n",
    "         tf.lgamma(S_alpha) - tf.reduce_sum(tf.lgamma(alpha), axis=1, keepdims=True) + \\\n",
    "         tf.reduce_sum(tf.lgamma(beta), axis=1, keepdims=True) - tf.lgamma(tf.reduce_sum(beta, axis=1, keepdims=True))\n",
    "    return KL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d8442ed3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_eq4(p, alpha, K, global_step, annealing_step, v):\n",
    "    loglikelihood = tf.reduce_mean(\n",
    "    tf.reduce_sum(p * (tf.digamma(tf.reduce_sum(alpha, axis=1, keepdims=True)) - tf.digamma(alpha)), 1,\n",
    "                    keepdims=True))\n",
    "    KL_reg = tf.minimum(1.0, tf.cast(global_step / annealing_step, tf.float32)) * KL((alpha - 1) * (1 - p) + 1, K)\n",
    "    loss = loglikelihood + KL_reg  # UAS_full\n",
    "    final_loss = tf.multiply(loss, v) #UAS_exp\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "69a17034",
   "metadata": {},
   "outputs": [],
   "source": [
    "bsize = 100 #batch size\n",
    "n_batches = np.ceil(1.0 * train_set.num_examples / bsize).astype(np.int32)\n",
    "n_batches_1 = np.ceil(1.0 * test_set.num_examples / bsize).astype(np.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "43ce4e5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_shd(epoch):\n",
    "    if epoch >=0 and epoch <=15:\n",
    "        lr = 0.001\n",
    "    else:\n",
    "        lr = 0.0001\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "039f8f9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Resent_EDL(units=10,logits2evidence=softplus_evidence,activation='elu',reg_factor=50e-4, \n",
    "               bias_reg_factor=None, batch_norm=False,loss_function=loss_eq4,lmb=0.005):\n",
    "    g = tf.Graph()\n",
    "    with g.as_default():\n",
    "        \n",
    "        X = tf.placeholder(shape=[None,32,32,3], dtype=tf.float32)\n",
    "        Y = tf.placeholder(shape=[None,10], dtype=tf.float32)\n",
    "        keep_prob_1 = tf.placeholder(dtype=tf.float32)\n",
    "        keep_prob_2 = tf.placeholder(dtype=tf.float32)\n",
    "        global_step = tf.Variable(initial_value=0, name='global_step', trainable=False)\n",
    "        annealing_step = tf.placeholder(dtype=tf.int32) \n",
    "        v = tf.placeholder(dtype=tf.float32)\n",
    "        lr = tf.placeholder(dtype=tf.float32)\n",
    "        \n",
    "        logits, evidence,W3,W4 = network(X, units=10,logits2evidence=softplus_evidence,activation='elu', dropout_1_rate=keep_prob_1, \n",
    "                                          dropout_2_rate=keep_prob_2,reg_factor=50e-4, bias_reg_factor=None, batch_norm=False)   \n",
    "        alpha = evidence + 1        \n",
    "        u = K / tf.reduce_sum(alpha, axis=1, keepdims=True) #uncertainty\n",
    "        prob = alpha/tf.reduce_sum(alpha, 1, keepdims=True) \n",
    "        \n",
    "        loss = tf.reduce_mean(loss_function(Y, alpha,K ,global_step, annealing_step, v))\n",
    "        l2_loss = (tf.nn.l2_loss(W3)+tf.nn.l2_loss(W4)) * lmb\n",
    "        total_loss = loss + l2_loss\n",
    "        step = tf.train.AdamOptimizer(lr).minimize(loss, global_step=global_step)\n",
    "        \n",
    "        # Calculate accuracy\n",
    "        pred = tf.argmax(prob, 1)\n",
    "        truth = tf.argmax(Y, 1)\n",
    "        match = tf.reshape(tf.cast(tf.equal(pred, truth), tf.float32),(-1,1))\n",
    "        acc = tf.reduce_mean(match)\n",
    "\n",
    "        \n",
    "        return g, step, X, Y, v, annealing_step, prob, acc, loss, u, keep_prob_1, keep_prob_2,lr, evidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2d223b6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "g2, step2, X2, Y2, v2, annealing_step, prob2, acc2, loss2,  u, keep_prob_1,keep_prob_2,lr,evidence = Resent_EDL()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "716e9d0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sess2 = tf.Session(graph=g2)\n",
    "with g2.as_default():\n",
    "    sess2.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "89d5ffe7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 - 100%) training:  acc: 0.4248 loss: 1.7251 \t testing: acc: 0.5632\n",
      "best epoch: 1 best acc: 0.5632\n",
      "epoch 2 - 100%) training:  acc: 0.6519 loss: 1.2742 \t testing: acc: 0.6847\n",
      "best epoch: 2 best acc: 0.6847\n",
      "epoch 3 - 100%) training:  acc: 0.7392 loss: 1.0798 \t testing: acc: 0.7364\n",
      "best epoch: 3 best acc: 0.7364\n",
      "epoch 4 - 100%) training:  acc: 0.7845 loss: 0.9762 \t testing: acc: 0.7627\n",
      "best epoch: 4 best acc: 0.7627\n",
      "epoch 5 - 100%) training:  acc: 0.8220 loss: 0.8646 \t testing: acc: 0.7712\n",
      "best epoch: 5 best acc: 0.7712\n",
      "epoch 6 - 100%) training:  acc: 0.8403 loss: 0.8290 \t testing: acc: 0.7855\n",
      "best epoch: 6 best acc: 0.7855\n",
      "epoch 7 - 100%) training:  acc: 0.8564 loss: 0.7862 \t testing: acc: 0.7711\n",
      "best epoch: 6 best acc: 0.7855\n",
      "epoch 8 - 100%) training:  acc: 0.8790 loss: 0.7009 \t testing: acc: 0.7918\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 9 - 100%) training:  acc: 0.8848 loss: 0.6930 \t testing: acc: 0.7770\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 10 - 100%) training:  acc: 0.8948 loss: 0.6586 \t testing: acc: 0.7801\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 11 - 100%) training:  acc: 0.9032 loss: 0.6169 \t testing: acc: 0.7758\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 12 - 100%) training:  acc: 0.8969 loss: 0.6762 \t testing: acc: 0.7717\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 13 - 100%) training:  acc: 0.8956 loss: 0.6852 \t testing: acc: 0.7530\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 14 - 100%) training:  acc: 0.8996 loss: 0.6693 \t testing: acc: 0.7683\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 15 - 100%) training:  acc: 0.9047 loss: 0.6412 \t testing: acc: 0.7716\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 16 - 100%) training:  acc: 0.8983 loss: 0.6963 \t testing: acc: 0.7582\n",
      "best epoch: 8 best acc: 0.7918\n",
      "epoch 17 - 100%) training:  acc: 0.9473 loss: 0.4046 \t testing: acc: 0.8086\n",
      "best epoch: 17 best acc: 0.8086\n",
      "epoch 18 - 100%) training:  acc: 0.9630 loss: 0.2781 \t testing: acc: 0.8126\n",
      "best epoch: 18 best acc: 0.8126\n",
      "epoch 19 - 100%) training:  acc: 0.9675 loss: 0.2290 \t testing: acc: 0.8103\n",
      "best epoch: 18 best acc: 0.8126\n",
      "epoch 20 - 100%) training:  acc: 0.9708 loss: 0.2010 \t testing: acc: 0.8166\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 21 - 100%) training:  acc: 0.9723 loss: 0.1824 \t testing: acc: 0.8120\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 22 - 100%) training:  acc: 0.9740 loss: 0.1700 \t testing: acc: 0.8142\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 23 - 100%) training:  acc: 0.9745 loss: 0.1617 \t testing: acc: 0.8116\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 24 - 100%) training:  acc: 0.9748 loss: 0.1580 \t testing: acc: 0.8084\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 25 - 100%) training:  acc: 0.9757 loss: 0.1455 \t testing: acc: 0.8037\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 26 - 100%) training:  acc: 0.9764 loss: 0.1441 \t testing: acc: 0.8013\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 27 - 100%) training:  acc: 0.9768 loss: 0.1375 \t testing: acc: 0.8029\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 28 - 100%) training:  acc: 0.9771 loss: 0.1374 \t testing: acc: 0.7939\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 29 - 100%) training:  acc: 0.9775 loss: 0.1308 \t testing: acc: 0.7987\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 30 - 100%) training:  acc: 0.9776 loss: 0.1327 \t testing: acc: 0.7878\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 31 - 100%) training:  acc: 0.9779 loss: 0.1298 \t testing: acc: 0.7910\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 32 - 100%) training:  acc: 0.9782 loss: 0.1254 \t testing: acc: 0.7891\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 33 - 100%) training:  acc: 0.9786 loss: 0.1196 \t testing: acc: 0.7877\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 34 - 100%) training:  acc: 0.9789 loss: 0.1186 \t testing: acc: 0.7840\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 35 - 100%) training:  acc: 0.9789 loss: 0.1174 \t testing: acc: 0.7782\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 36 - 100%) training:  acc: 0.9794 loss: 0.1154 \t testing: acc: 0.7848\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 37 - 100%) training:  acc: 0.9791 loss: 0.1143 \t testing: acc: 0.7838\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 38 - 100%) training:  acc: 0.9797 loss: 0.1084 \t testing: acc: 0.7806\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 39 - 100%) training:  acc: 0.9793 loss: 0.1217 \t testing: acc: 0.7732\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 40 - 100%) training:  acc: 0.9798 loss: 0.1079 \t testing: acc: 0.7792\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 41 - 100%) training:  acc: 0.9797 loss: 0.1145 \t testing: acc: 0.7789\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 42 - 100%) training:  acc: 0.9799 loss: 0.1081 \t testing: acc: 0.7784\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 43 - 100%) training:  acc: 0.9800 loss: 0.1089 \t testing: acc: 0.7811\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 44 - 100%) training:  acc: 0.9800 loss: 0.1097 \t testing: acc: 0.7746\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 45 - 100%) training:  acc: 0.9807 loss: 0.1057 \t testing: acc: 0.7884\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 46 - 100%) training:  acc: 0.9800 loss: 0.1144 \t testing: acc: 0.7709\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 47 - 100%) training:  acc: 0.9800 loss: 0.1081 \t testing: acc: 0.7818\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 48 - 100%) training:  acc: 0.9803 loss: 0.0989 \t testing: acc: 0.7863\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 49 - 100%) training:  acc: 0.9810 loss: 0.1015 \t testing: acc: 0.7934\n",
      "best epoch: 20 best acc: 0.8166\n",
      "epoch 50 - 100%) training:  acc: 0.9803 loss: 0.1074 \t testing: acc: 0.7846\n",
      "best epoch: 20 best acc: 0.8166\n"
     ]
    }
   ],
   "source": [
    "L_train_acc1=[]\n",
    "L_test_acc1=[]\n",
    "\n",
    "\n",
    "best_acc = 0\n",
    "for epoch in range(50):  \n",
    "    loss_trains = []\n",
    "    acc_train = []\n",
    "    acc_test = []\n",
    "    uncertainty = np.zeros(train_set.num_examples)\n",
    "    learning_rate = lr_shd(epoch)\n",
    "    train_set.train_order()\n",
    "    train_set.subset_step_indicate(epoch,increase_amount=1.2)\n",
    "    \n",
    "\n",
    "    start = 0\n",
    "    end = bsize\n",
    "    for i in range(n_batches):\n",
    "        data, label, probs, indices = train_set.next_batch_train(bsize)\n",
    "        feed_dict={X2:data, Y2:label, v2:indices, keep_prob_1:0.25, keep_prob_2:0.5, lr:learning_rate, annealing_step:50*n_batches}\n",
    "        _, train_loss, unc, train_acc1 = sess2.run([step2,loss2,u,acc2],feed_dict)\n",
    "        loss_trains.append(train_loss)\n",
    "        acc_train.append(train_acc1)\n",
    "        print('epoch %d - %d%%) '% (epoch+1, (100*(i+1))//n_batches), end='\\r' if i<n_batches-1 else '')\n",
    "        unc = np.array(unc).reshape((-1,))\n",
    "        if end > train_set.num_examples:\n",
    "            end = train_set.num_examples\n",
    "            unc = unc[:(end-start)]\n",
    "            \n",
    "        uncertainty[start:end] = unc\n",
    "        start += data.shape[0]\n",
    "        end += data.shape[0]\n",
    "    train_set.change_probs(uncertainty)\n",
    "    train_set.revert_indicte()\n",
    "    \n",
    "    for m in range(n_batches_1):\n",
    "        data2, label2, probs2, indices2 = test_set.next_batch_test(bsize)\n",
    "        feed_dict={X2:data2, Y2:label2,keep_prob_1:1.,keep_prob_2:1.}\n",
    "        test_acc1 = sess2.run(acc2, feed_dict)\n",
    "        acc_test.append(test_acc1)\n",
    " \n",
    "    \n",
    "    \n",
    "    \n",
    "    loss_train = np.mean(loss_trains)\n",
    "    train_acc = np.mean(acc_train)\n",
    "    test_acc = np.mean(acc_test)\n",
    "\n",
    "    \n",
    "    L_train_acc1.append(train_acc)\n",
    "    L_test_acc1.append(test_acc)\n",
    "\n",
    "    print('training:  acc: %2.4f loss: %2.4f \\t testing: acc: %2.4f' % \n",
    "          (train_acc, loss_train, test_acc))\n",
    "    \n",
    "    if test_acc > best_acc:\n",
    "        best_acc = test_acc\n",
    "        best_epoch = epoch + 1\n",
    "    print('best epoch: %d best acc: %2.4f' % (best_epoch, best_acc))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9c73600",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}