From 74a826fe25e0051c6c60ec452acfc87848be2bb7 Mon Sep 17 00:00:00 2001 From: Marc Masana Date: Mon, 1 Mar 2021 09:15:20 +0100 Subject: [PATCH] Remaining alpha release files Alpha release Co-authored-by: Mikel Menta Garde Co-authored-by: Bartlomiej Twardowski Co-authored-by: Xialei Liu --- src/README.md | 61 +++++ src/gridsearch.py | 122 ++++++++++ src/gridsearch_config.py | 73 ++++++ src/last_layer_analysis.py | 61 +++++ src/loggers/README.md | 35 +++ src/loggers/disk_logger.py | 75 ++++++ src/loggers/exp_logger.py | 75 ++++++ src/loggers/tensorboard_logger.py | 52 +++++ src/main_incremental.py | 316 ++++++++++++++++++++++++++ src/networks/README.md | 58 +++++ src/networks/__init__.py | 43 ++++ src/networks/lenet.py | 30 +++ src/networks/network.py | 95 ++++++++ src/networks/resnet32.py | 115 ++++++++++ src/networks/vggnet.py | 58 +++++ src/tests/README.md | 33 +++ src/tests/__init__.py | 52 +++++ src/tests/test_bic.py | 34 +++ src/tests/test_dataloader.py | 42 ++++ src/tests/test_datasets_transforms.py | 35 +++ src/tests/test_dmc.py | 13 ++ src/tests/test_eeil.py | 31 +++ src/tests/test_ewc.py | 25 ++ src/tests/test_finetuning.py | 78 +++++++ src/tests/test_fix_bn.py | 80 +++++++ src/tests/test_freezing.py | 24 ++ src/tests/test_gridsearch.py | 92 ++++++++ src/tests/test_icarl.py | 30 +++ src/tests/test_il2m.py | 21 ++ src/tests/test_joint.py | 12 + src/tests/test_last_layer_analysis.py | 14 ++ src/tests/test_loggers.py | 31 +++ src/tests/test_lucir.py | 49 ++++ src/tests/test_lwf.py | 25 ++ src/tests/test_lwm.py | 27 +++ src/tests/test_mas.py | 25 ++ src/tests/test_multisoftmax.py | 19 ++ src/tests/test_path_integral.py | 25 ++ src/tests/test_rwalk.py | 27 +++ src/tests/test_stop_at_task.py | 11 + src/tests/test_warmup.py | 20 ++ src/utils.py | 34 +++ 42 files changed, 2178 insertions(+) create mode 100644 src/README.md create mode 100644 src/gridsearch.py create mode 100644 src/gridsearch_config.py create mode 100644 src/last_layer_analysis.py create mode 100644 src/loggers/README.md create mode 100644 src/loggers/disk_logger.py create mode 100644 src/loggers/exp_logger.py create mode 100644 src/loggers/tensorboard_logger.py create mode 100644 src/main_incremental.py create mode 100644 src/networks/README.md create mode 100644 src/networks/__init__.py create mode 100644 src/networks/lenet.py create mode 100644 src/networks/network.py create mode 100644 src/networks/resnet32.py create mode 100644 src/networks/vggnet.py create mode 100644 src/tests/README.md create mode 100644 src/tests/__init__.py create mode 100644 src/tests/test_bic.py create mode 100644 src/tests/test_dataloader.py create mode 100644 src/tests/test_datasets_transforms.py create mode 100644 src/tests/test_dmc.py create mode 100644 src/tests/test_eeil.py create mode 100644 src/tests/test_ewc.py create mode 100644 src/tests/test_finetuning.py create mode 100644 src/tests/test_fix_bn.py create mode 100644 src/tests/test_freezing.py create mode 100644 src/tests/test_gridsearch.py create mode 100644 src/tests/test_icarl.py create mode 100644 src/tests/test_il2m.py create mode 100644 src/tests/test_joint.py create mode 100644 src/tests/test_last_layer_analysis.py create mode 100644 src/tests/test_loggers.py create mode 100644 src/tests/test_lucir.py create mode 100644 src/tests/test_lwf.py create mode 100644 src/tests/test_lwm.py create mode 100644 src/tests/test_mas.py create mode 100644 src/tests/test_multisoftmax.py create mode 100644 src/tests/test_path_integral.py create mode 100644 src/tests/test_rwalk.py create mode 100644 src/tests/test_stop_at_task.py create mode 100644 src/tests/test_warmup.py create mode 100644 src/utils.py diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..5e255bb --- /dev/null +++ b/src/README.md @@ -0,0 +1,61 @@ +# Framework for Analysis of Class-Incremental Learning +Run the code with: +``` +python3 -u src/main_incremental.py +``` +followed by general options: + +* `--gpu`: index of GPU to run the experiment on (default=0) +* `--results-path`: path where results are stored (default='../results') +* `--exp-name`: experiment name (default=None) +* `--seed`: random seed (default=0) +* `--save-models`: save trained models (default=False) +* `--last-layer-analysis`: plot last layer analysis (default=False) +* `--no-cudnn-deterministic`: disable CUDNN deterministic (default=False) + +and specific options for each of the code parts (corresponding to folders): + +* `--approach`: learning approach used (default='finetuning') [[more](approaches/README.md)] +* `--datasets`: dataset or datasets used (default=['cifar100']) [[more](datasets/README.md)] +* `--network`: network architecture used (default='resnet32') [[more](networks/README.md)] +* `--log`: loggers used (default='disk') [[more](loggers/README.md)] + +go to each of their respective readme to see all available options for each of them. + +## Approaches +Initially, the approaches included in the framework correspond to the ones presented in +_**Class-incremental learning: survey and performance evaluation**_ (preprint , 2020). The regularization-based +approaches are EWC, MAS, PathInt, LwF, LwM and DMC (green). The rehearsal approaches are iCaRL, EEIL and RWalk (blue). +The bias-correction approaches are IL2M, BiC and LUCIR (orange). + +![alt text](../docs/_static/cil_survey_approaches.png "Survey approaches") + +More approaches will be included in the future. To learn more about them refer to the readme in +[src/approaches](approaches). + +## Datasets +To learn about the dataset management refer to the readme in [src/datasets](datasets). + +## Networks +To learn about the different torchvision and custom networks refer to the readme in [src/networks](networks). + +## GridSearch +We implement the option to use a realistic grid search for hyperparameters which only takes into account the task at +hand, without access to previous or future information not available in the incremental learning scenario. It +corresponds to the one introduced in _**Class-incremental learning: survey and performance evaluation**_. The GridSearch +can be applied by using: + +* `--gridsearch-tasks`: number of tasks to apply GridSearch (-1: all tasks) (default=-1) + +which we recommend to set to the total number of tasks of the experiment for a more realistic setting of the correct +learning rate and possible forgetting-intransigence trade-off. However, since this has a considerable extra +computational cost, it can also be set to the first 3 tasks, which would fix those hyperparameters for the remaining +tasks. Other GridSearch options include: + +* `--gridsearch-config`: configuration file for GridSearch options (default='gridsearch_config') [[more](gridsearch_config.py)] +* `--gridsearch-acc-drop-thr`: GridSearch accuracy drop threshold (default=0.2) +* `--gridsearch-hparam-decay`: GridSearch hyperparameter decay (default=0.5) +* `--gridsearch-max-num-searches`: GridSearch maximum number of hyperparameter search (default=7) + +## Utils +We have some utility functions added into `utils.py`. \ No newline at end of file diff --git a/src/gridsearch.py b/src/gridsearch.py new file mode 100644 index 0000000..ab5adc7 --- /dev/null +++ b/src/gridsearch.py @@ -0,0 +1,122 @@ +import importlib +from copy import deepcopy +from argparse import ArgumentParser + +import utils + + +class GridSearch: + """Basic class for implementing hyperparameter grid search""" + + def __init__(self, appr_ft, seed, gs_config='gridsearch_config', acc_drop_thr=0.2, hparam_decay=0.5, + max_num_searches=7): + self.seed = seed + GridSearchConfig = getattr(importlib.import_module(name=gs_config), 'GridSearchConfig') + self.appr_ft = appr_ft + self.gs_config = GridSearchConfig() + self.acc_drop_thr = acc_drop_thr + self.hparam_decay = hparam_decay + self.max_num_searches = max_num_searches + self.lr_first = 1.0 + + @staticmethod + def extra_parser(args): + """Returns a parser containing the GridSearch specific parameters""" + parser = ArgumentParser() + # Configuration file with a GridSearchConfig class with all necessary args + parser.add_argument('--gridsearch-config', type=str, default='gridsearch_config', required=False, + help='Configuration file for GridSearch options (default=%(default)s)') + # Accuracy threshold drop below which the search stops for that phase + parser.add_argument('--gridsearch-acc-drop-thr', default=0.2, type=float, required=False, + help='GridSearch accuracy drop threshold (default=%(default)f)') + # Value at which hyperparameters decay + parser.add_argument('--gridsearch-hparam-decay', default=0.5, type=float, required=False, + help='GridSearch hyperparameter decay (default=%(default)f)') + # Maximum number of searched before the search stops for that phase + parser.add_argument('--gridsearch-max-num-searches', default=7, type=int, required=False, + help='GridSearch maximum number of hyperparameter search (default=%(default)f)') + return parser.parse_known_args(args) + + def search_lr(self, model, t, trn_loader, val_loader): + """Search for accuracy and best LR on finetuning""" + best_ft_acc = 0.0 + best_ft_lr = 0.0 + + # Get general parameters and fix the ones with only one value + gen_params = self.gs_config.get_params('general') + for k, v in gen_params.items(): + if not isinstance(v, list): + setattr(self.appr_ft, k, v) + if t > 0: + # LR for search are 'lr_searches' largest LR below 'lr_first' + list_lr = [lr for lr in gen_params['lr'] if lr < self.lr_first][:gen_params['lr_searches'][0]] + else: + # For first task, try larger LR range + list_lr = gen_params['lr_first'] + + # Iterate through the other variable parameters + for curr_lr in list_lr: + utils.seed_everything(seed=self.seed) + self.appr_ft.model = deepcopy(model) + self.appr_ft.lr = curr_lr + self.appr_ft.train(t, trn_loader, val_loader) + _, ft_acc_taw, _ = self.appr_ft.eval(t, val_loader) + if ft_acc_taw > best_ft_acc: + best_ft_acc = ft_acc_taw + best_ft_lr = curr_lr + print('Current best LR: ' + str(best_ft_lr)) + self.gs_config.current_lr = best_ft_lr + print('Current best acc: {:5.1f}'.format(best_ft_acc * 100)) + # After first task, keep LR used + if t == 0: + self.lr_first = best_ft_lr + + return best_ft_acc, best_ft_lr + + def search_tradeoff(self, appr_name, appr, t, trn_loader, val_loader, best_ft_acc): + """Search for less-forgetting tradeoff with minimum accuracy loss""" + best_tradeoff = None + tradeoff_name = None + + # Get general parameters and fix all the ones that have only one option + appr_params = self.gs_config.get_params(appr_name) + for k, v in appr_params.items(): + if isinstance(v, list): + # get tradeoff name as the only one with multiple values + tradeoff_name = k + else: + # Any other hyperparameters are fixed + setattr(appr, k, v) + + # If there is no tradeoff, no need to gridsearch more + if tradeoff_name is not None and t > 0: + # get starting value for trade-off hyperparameter + best_tradeoff = appr_params[tradeoff_name][0] + # iterate through decreasing trade-off values -- limit to `max_num_searches` searches + num_searches = 0 + while num_searches < self.max_num_searches: + utils.seed_everything(seed=self.seed) + # Make deepcopy of the appr without duplicating the logger + appr_gs = type(appr)(deepcopy(appr.model), appr.device, exemplars_dataset=appr.exemplars_dataset) + for attr, value in vars(appr).items(): + if attr == 'logger': + setattr(appr_gs, attr, value) + else: + setattr(appr_gs, attr, deepcopy(value)) + + # update tradeoff value + setattr(appr_gs, tradeoff_name, best_tradeoff) + # train this iteration + appr_gs.train(t, trn_loader, val_loader) + _, curr_acc, _ = appr_gs.eval(t, val_loader) + print('Current acc: ' + str(curr_acc) + ' for ' + tradeoff_name + '=' + str(best_tradeoff)) + # Check if accuracy is within acceptable threshold drop + if curr_acc < ((1 - self.acc_drop_thr) * best_ft_acc): + best_tradeoff = best_tradeoff * self.hparam_decay + else: + break + num_searches += 1 + else: + print('There is no trade-off to gridsearch.') + + return best_tradeoff, tradeoff_name diff --git a/src/gridsearch_config.py b/src/gridsearch_config.py new file mode 100644 index 0000000..7efe2a0 --- /dev/null +++ b/src/gridsearch_config.py @@ -0,0 +1,73 @@ +class GridSearchConfig(): + def __init__(self): + self.params = { + 'general': { + 'lr_first': [5e-1, 1e-1, 5e-2], + 'lr': [1e-1, 5e-2, 1e-2, 5e-3, 1e-3], + 'lr_searches': [3], + 'lr_min': 1e-4, + 'lr_factor': 3, + 'lr_patience': 10, + 'clipping': 10000, + 'momentum': 0.9, + 'wd': 0.0002 + }, + 'finetuning': { + }, + 'freezing': { + }, + 'joint': { + }, + 'lwf': { + 'lamb': [10], + 'T': 2 + }, + 'icarl': { + 'lamb': [4] + }, + 'dmc': { + 'aux_dataset': 'imagenet_32_reduced', + 'aux_batch_size': 128 + }, + 'il2m': { + }, + 'eeil': { + 'lamb': [10], + 'T': 2, + 'lr_finetuning_factor': 0.1, + 'nepochs_finetuning': 40, + 'noise_grad': False + }, + 'bic': { + 'T': 2, + 'val_percentage': 0.1, + 'bias_epochs': 200 + }, + 'lucir': { + 'lamda_base': [10], + 'lamda_mr': 1.0, + 'dist': 0.5, + 'K': 2 + }, + 'lwm': { + 'beta': [2], + 'gamma': 1.0 + }, + 'ewc': { + 'lamb': [10000] + }, + 'mas': { + 'lamb': [400] + }, + 'path_integral': { + 'lamb': [10], + }, + 'r_walk': { + 'lamb': [20], + }, + } + self.current_lr = self.params['general']['lr'][0] + self.current_tradeoff = 0 + + def get_params(self, approach): + return self.params[approach] diff --git a/src/last_layer_analysis.py b/src/last_layer_analysis.py new file mode 100644 index 0000000..7f157ec --- /dev/null +++ b/src/last_layer_analysis.py @@ -0,0 +1,61 @@ +import torch +import matplotlib +import numpy as np +import matplotlib.pyplot as plt +matplotlib.use('Agg') + + +def last_layer_analysis(heads, task, taskcla, y_lim=False, sort_weights=False): + """Plot last layer weight and bias analysis""" + print('Plotting last layer analysis...') + num_classes = sum([x for (_, x) in taskcla]) + weights, biases, indexes = [], [], [] + class_id = 0 + with torch.no_grad(): + for t in range(task + 1): + n_classes_t = taskcla[t][1] + indexes.append(np.arange(class_id, class_id + n_classes_t)) + if type(heads) == torch.nn.Linear: # Single head + biases.append(heads.bias[class_id: class_id + n_classes_t].detach().cpu().numpy()) + weights.append((heads.weight[class_id: class_id + n_classes_t] ** 2).sum(1).sqrt().detach().cpu().numpy()) + else: # Multi-head + weights.append((heads[t].weight ** 2).sum(1).sqrt().detach().cpu().numpy()) + if type(heads[t]) == torch.nn.Linear: + biases.append(heads[t].bias.detach().cpu().numpy()) + else: + biases.append(np.zeros(weights[-1].shape)) # For LUCIR + class_id += n_classes_t + + # Figure weights + f_weights = plt.figure(dpi=300) + ax = f_weights.subplots(nrows=1, ncols=1) + for i, (x, y) in enumerate(zip(indexes, weights), 0): + if sort_weights: + ax.bar(x, sorted(y, reverse=True), label="Task {}".format(i)) + else: + ax.bar(x, y, label="Task {}".format(i)) + ax.set_xlabel("Classes", fontsize=11, fontfamily='serif') + ax.set_ylabel("Weights L2-norm", fontsize=11, fontfamily='serif') + if num_classes is not None: + ax.set_xlim(0, num_classes) + if y_lim: + ax.set_ylim(0, 5) + ax.legend(loc='upper left', fontsize='11') #, fontfamily='serif') + + # Figure biases + f_biases = plt.figure(dpi=300) + ax = f_biases.subplots(nrows=1, ncols=1) + for i, (x, y) in enumerate(zip(indexes, biases), 0): + if sort_weights: + ax.bar(x, sorted(y, reverse=True), label="Task {}".format(i)) + else: + ax.bar(x, y, label="Task {}".format(i)) + ax.set_xlabel("Classes", fontsize=11, fontfamily='serif') + ax.set_ylabel("Bias values", fontsize=11, fontfamily='serif') + if num_classes is not None: + ax.set_xlim(0, num_classes) + if y_lim: + ax.set_ylim(-1.0, 1.0) + ax.legend(loc='upper left', fontsize='11') #, fontfamily='serif') + + return f_weights, f_biases diff --git a/src/loggers/README.md b/src/loggers/README.md new file mode 100644 index 0000000..02390c8 --- /dev/null +++ b/src/loggers/README.md @@ -0,0 +1,35 @@ +# Loggers + +We include a disk logger, which logs into files and folders in the disk. We also provide a tensorboard logger which +provides a faster way of analysing a training process without need of further development. They can be specified with +`--log` followed by `disk`, `tensorboard` or both. Custom loggers can be defined by inheriting the `ExperimentLogger` +in [exp_logger.py](exp_logger.py). + +When enabled, both loggers will output everything in the path `[RESULTS_PATH]/[DATASETS]_[APPROACH]_[EXP_NAME]` or +`[RESULTS_PATH]/[DATASETS]_[APPROACH]` if `--exp-name` is not set. + +## Disk logger +The disk logger outputs the following file and folder structure: +- **figures/**: folder where generated figures are logged. +- **models/**: folder where model weight checkpoints are saved. +- **results/**: folder containing the results. + - **acc_tag**: task-agnostic accuracy table. + - **acc_taw**: task-aware accuracy table. + - **avg_acc_tag**: task-agnostic average accuracies. + - **avg_acc_taw**: task-agnostic average accuracies. + - **forg_tag**: task-agnostic forgetting table. + - **forg_taw**: task-aware forgetting table. + - **wavg_acc_tag**: task-agnostic average accuracies weighted according to the number of classes of each task. + - **wavg_acc_taw**: task-aware average accuracies weighted according to the number of classes of each task. +- **raw_log**: json file containing all the logged metrics easily read by many tools (e.g. `pandas`). +- stdout: a copy from the standard output of the terminal. +- stderr: a copy from the error output of the terminal. + +## TensorBoard logger +The tensorboard logger outputs analogous metrics to the disk logger separated into different tabs according to the task +and different graphs according to the data splits. + +Screenshot for a 10 task experiment, showing the last task plots: +

+Tensorboard Screenshot +

diff --git a/src/loggers/disk_logger.py b/src/loggers/disk_logger.py new file mode 100644 index 0000000..7b9a4a2 --- /dev/null +++ b/src/loggers/disk_logger.py @@ -0,0 +1,75 @@ +import os +import sys +import json +import torch +import numpy as np +from datetime import datetime + +from loggers.exp_logger import ExperimentLogger + + +class Logger(ExperimentLogger): + """Characterizes a disk logger""" + + def __init__(self, log_path, exp_name, begin_time=None): + super(Logger, self).__init__(log_path, exp_name, begin_time) + + self.begin_time_str = self.begin_time.strftime("%Y-%m-%d-%H-%M") + + # Duplicate standard outputs + sys.stdout = FileOutputDuplicator(sys.stdout, + os.path.join(self.exp_path, 'stdout-{}.txt'.format(self.begin_time_str)), 'w') + sys.stderr = FileOutputDuplicator(sys.stderr, + os.path.join(self.exp_path, 'stderr-{}.txt'.format(self.begin_time_str)), 'w') + + # Raw log file + self.raw_log_file = open(os.path.join(self.exp_path, "raw_log-{}.txt".format(self.begin_time_str)), 'a') + + def log_scalar(self, task, iter, name, value, group=None, curtime=None): + if curtime is None: + curtime = datetime.now() + + # Raw dump + entry = {"task": task, "iter": iter, "name": name, "value": value, "group": group, + "time": curtime.strftime("%Y-%m-%d-%H-%M")} + self.raw_log_file.write(json.dumps(entry, sort_keys=True) + "\n") + self.raw_log_file.flush() + + def log_args(self, args): + with open(os.path.join(self.exp_path, 'args-{}.txt'.format(self.begin_time_str)), 'w') as f: + json.dump(args.__dict__, f, separators=(',\n', ' : '), sort_keys=True) + + def log_result(self, array, name, step): + if array.ndim <= 1: + array = array[None] + np.savetxt(os.path.join(self.exp_path, 'results', '{}-{}.txt'.format(name, self.begin_time_str)), + array, '%.6f', delimiter='\t') + + def log_figure(self, name, iter, figure, curtime=None): + curtime = datetime.now() + figure.savefig(os.path.join(self.exp_path, 'figures', + '{}_{}-{}.png'.format(name, iter, curtime.strftime("%Y-%m-%d-%H-%M-%S")))) + figure.savefig(os.path.join(self.exp_path, 'figures', + '{}_{}-{}.pdf'.format(name, iter, curtime.strftime("%Y-%m-%d-%H-%M-%S")))) + + def save_model(self, state_dict, task): + torch.save(state_dict, os.path.join(self.exp_path, "models", "task{}.ckpt".format(task))) + + def __del__(self): + self.raw_log_file.close() + + +class FileOutputDuplicator(object): + def __init__(self, duplicate, fname, mode): + self.file = open(fname, mode) + self.duplicate = duplicate + + def __del__(self): + self.file.close() + + def write(self, data): + self.file.write(data) + self.duplicate.write(data) + + def flush(self): + self.file.flush() diff --git a/src/loggers/exp_logger.py b/src/loggers/exp_logger.py new file mode 100644 index 0000000..b8d875f --- /dev/null +++ b/src/loggers/exp_logger.py @@ -0,0 +1,75 @@ +import os +import importlib +from datetime import datetime + + +class ExperimentLogger: + """Main class for experiment logging""" + + def __init__(self, log_path, exp_name, begin_time=None): + self.log_path = log_path + self.exp_name = exp_name + self.exp_path = os.path.join(log_path, exp_name) + if begin_time is None: + self.begin_time = datetime.now() + else: + self.begin_time = begin_time + + def log_scalar(self, task, iter, name, value, group=None, curtime=None): + pass + + def log_args(self, args): + pass + + def log_result(self, array, name, step): + pass + + def log_figure(self, name, iter, figure, curtime=None): + pass + + def save_model(self, state_dict, task): + pass + + +class MultiLogger(ExperimentLogger): + """This class allows to use multiple loggers""" + + def __init__(self, log_path, exp_name, loggers=None, save_models=True): + super(MultiLogger, self).__init__(log_path, exp_name) + if os.path.exists(self.exp_path): + print("WARNING: {} already exists!".format(self.exp_path)) + else: + os.makedirs(os.path.join(self.exp_path, 'models')) + os.makedirs(os.path.join(self.exp_path, 'results')) + os.makedirs(os.path.join(self.exp_path, 'figures')) + + self.save_models = save_models + self.loggers = [] + for l in loggers: + lclass = getattr(importlib.import_module(name='loggers.' + l + '_logger'), 'Logger') + self.loggers.append(lclass(self.log_path, self.exp_name)) + + def log_scalar(self, task, iter, name, value, group=None, curtime=None): + if curtime is None: + curtime = datetime.now() + for l in self.loggers: + l.log_scalar(task, iter, name, value, group, curtime) + + def log_args(self, args): + for l in self.loggers: + l.log_args(args) + + def log_result(self, array, name, step): + for l in self.loggers: + l.log_result(array, name, step) + + def log_figure(self, name, iter, figure, curtime=None): + if curtime is None: + curtime = datetime.now() + for l in self.loggers: + l.log_figure(name, iter, figure, curtime) + + def save_model(self, state_dict, task): + if self.save_models: + for l in self.loggers: + l.save_model(state_dict, task) diff --git a/src/loggers/tensorboard_logger.py b/src/loggers/tensorboard_logger.py new file mode 100644 index 0000000..e05c9e5 --- /dev/null +++ b/src/loggers/tensorboard_logger.py @@ -0,0 +1,52 @@ +from torch.utils.tensorboard import SummaryWriter + +from loggers.exp_logger import ExperimentLogger +import json +import numpy as np + + +class Logger(ExperimentLogger): + """Characterizes a Tensorboard logger""" + + def __init__(self, log_path, exp_name, begin_time=None): + super(Logger, self).__init__(log_path, exp_name, begin_time) + self.tbwriter = SummaryWriter(self.exp_path) + + def log_scalar(self, task, iter, name, value, group=None, curtime=None): + self.tbwriter.add_scalar(tag="t{}/{}_{}".format(task, group, name), + scalar_value=value, + global_step=iter) + self.tbwriter.file_writer.flush() + + def log_figure(self, name, iter, figure, curtime=None): + self.tbwriter.add_figure(tag=name, figure=figure, global_step=iter) + self.tbwriter.file_writer.flush() + + def log_args(self, args): + self.tbwriter.add_text( + 'args', + json.dumps(args.__dict__, + separators=(',\n', ' : '), + sort_keys=True)) + self.tbwriter.file_writer.flush() + + def log_result(self, array, name, step): + if array.ndim == 1: + # log as scalars + self.tbwriter.add_scalar(f'results/{name}', array[step], step) + + elif array.ndim == 2: + s = "" + i = step + # for i in range(array.shape[0]): + for j in range(array.shape[1]): + s += '{:5.1f}% '.format(100 * array[i, j]) + if np.trace(array) == 0.0: + if i > 0: + s += '\tAvg.:{:5.1f}% \n'.format(100 * array[i, :i].mean()) + else: + s += '\tAvg.:{:5.1f}% \n'.format(100 * array[i, :i + 1].mean()) + self.tbwriter.add_text(f'results/{name}', s, step) + + def __del__(self): + self.tbwriter.close() diff --git a/src/main_incremental.py b/src/main_incremental.py new file mode 100644 index 0000000..490461e --- /dev/null +++ b/src/main_incremental.py @@ -0,0 +1,316 @@ +import os +import time +import torch +import argparse +import importlib +import numpy as np +from functools import reduce + +import utils +import approach +from loggers.exp_logger import MultiLogger +from datasets.data_loader import get_loaders +from datasets.dataset_config import dataset_config +from last_layer_analysis import last_layer_analysis +from networks import tvmodels, allmodels, set_tvmodel_head_var + + +def main(argv=None): + tstart = time.time() + # Arguments + parser = argparse.ArgumentParser(description='FACIL - Framework for Analysis of Class Incremental Learning') + + # miscellaneous args + parser.add_argument('--gpu', type=int, default=0, + help='GPU (default=%(default)s)') + parser.add_argument('--results-path', type=str, default='../results', + help='Results path (default=%(default)s)') + parser.add_argument('--exp-name', default=None, type=str, + help='Experiment name (default=%(default)s)') + parser.add_argument('--seed', type=int, default=0, + help='Random seed (default=%(default)s)') + parser.add_argument('--log', default=['disk'], type=str, choices=['disk', 'tensorboard'], + help='Loggers used (disk, tensorboard) (default=%(default)s)', nargs='*', metavar="LOGGER") + parser.add_argument('--save-models', action='store_true', + help='Save trained models (default=%(default)s)') + parser.add_argument('--last-layer-analysis', action='store_true', + help='Plot last layer analysis (default=%(default)s)') + parser.add_argument('--no-cudnn-deterministic', action='store_true', + help='Disable CUDNN deterministic (default=%(default)s)') + # dataset args + parser.add_argument('--datasets', default=['cifar100'], type=str, choices=list(dataset_config.keys()), + help='Dataset or datasets used (default=%(default)s)', nargs='+', metavar="DATASET") + parser.add_argument('--num-workers', default=4, type=int, required=False, + help='Number of subprocesses to use for dataloader (default=%(default)s)') + parser.add_argument('--pin-memory', default=False, type=bool, required=False, + help='Copy Tensors into CUDA pinned memory before returning them (default=%(default)s)') + parser.add_argument('--batch-size', default=64, type=int, required=False, + help='Number of samples per batch to load (default=%(default)s)') + parser.add_argument('--num-tasks', default=4, type=int, required=False, + help='Number of tasks per dataset (default=%(default)s)') + parser.add_argument('--nc-first-task', default=None, type=int, required=False, + help='Number of classes of the first task (default=%(default)s)') + parser.add_argument('--use-valid-only', action='store_true', + help='Use validation split instead of test (default=%(default)s)') + parser.add_argument('--stop-at-task', default=0, type=int, required=False, + help='Stop training after specified task (default=%(default)s)') + # model args + parser.add_argument('--network', default='resnet32', type=str, choices=allmodels, + help='Network architecture used (default=%(default)s)', metavar="NETWORK") + parser.add_argument('--keep-existing-head', action='store_true', + help='Disable removing classifier last layer (default=%(default)s)') + parser.add_argument('--pretrained', action='store_true', + help='Use pretrained backbone (default=%(default)s)') + # training args + parser.add_argument('--approach', default='finetuning', type=str, choices=approach.__all__, + help='Learning approach used (default=%(default)s)', metavar="APPROACH") + parser.add_argument('--nepochs', default=200, type=int, required=False, + help='Number of epochs per training session (default=%(default)s)') + parser.add_argument('--lr', default=0.1, type=float, required=False, + help='Starting learning rate (default=%(default)s)') + parser.add_argument('--lr-min', default=1e-4, type=float, required=False, + help='Minimum learning rate (default=%(default)s)') + parser.add_argument('--lr-factor', default=3, type=float, required=False, + help='Learning rate decreasing factor (default=%(default)s)') + parser.add_argument('--lr-patience', default=5, type=int, required=False, + help='Maximum patience to wait before decreasing learning rate (default=%(default)s)') + parser.add_argument('--clipping', default=10000, type=float, required=False, + help='Clip gradient norm (default=%(default)s)') + parser.add_argument('--momentum', default=0.0, type=float, required=False, + help='Momentum factor (default=%(default)s)') + parser.add_argument('--weight-decay', default=0.0, type=float, required=False, + help='Weight decay (L2 penalty) (default=%(default)s)') + parser.add_argument('--warmup-nepochs', default=0, type=int, required=False, + help='Number of warm-up epochs (default=%(default)s)') + parser.add_argument('--warmup-lr-factor', default=1.0, type=float, required=False, + help='Warm-up learning rate factor (default=%(default)s)') + parser.add_argument('--multi-softmax', action='store_true', + help='Apply separate softmax for each task (default=%(default)s)') + parser.add_argument('--fix-bn', action='store_true', + help='Fix batch normalization after first task (default=%(default)s)') + parser.add_argument('--eval-on-train', action='store_true', + help='Show train loss and accuracy (default=%(default)s)') + # gridsearch args + parser.add_argument('--gridsearch-tasks', default=-1, type=int, + help='Number of tasks to apply GridSearch (-1: all tasks) (default=%(default)s)') + + # Args -- Incremental Learning Framework + args, extra_args = parser.parse_known_args(argv) + args.results_path = os.path.expanduser(args.results_path) + base_kwargs = dict(nepochs=args.nepochs, lr=args.lr, lr_min=args.lr_min, lr_factor=args.lr_factor, + lr_patience=args.lr_patience, clipgrad=args.clipping, momentum=args.momentum, + wd=args.weight_decay, multi_softmax=args.multi_softmax, wu_nepochs=args.warmup_nepochs, + wu_lr_factor=args.warmup_lr_factor, fix_bn=args.fix_bn, eval_on_train=args.eval_on_train) + + if args.no_cudnn_deterministic: + print('WARNING: CUDNN Deterministic will be disabled.') + utils.cudnn_deterministic = False + + utils.seed_everything(seed=args.seed) + print('=' * 108) + print('Arguments =') + for arg in np.sort(list(vars(args).keys())): + print('\t' + arg + ':', getattr(args, arg)) + print('=' * 108) + + # Args -- CUDA + if torch.cuda.is_available(): + torch.cuda.set_device(args.gpu) + device = 'cuda' + else: + print('WARNING: [CUDA unavailable] Using CPU instead!') + device = 'cpu' + # Multiple gpus + # if torch.cuda.device_count() > 1: + # self.C = torch.nn.DataParallel(C) + # self.C.to(self.device) + #################################################################################################################### + + # Args -- Network + from networks.network import LLL_Net + if args.network in tvmodels: # torchvision models + tvnet = getattr(importlib.import_module(name='torchvision.models'), args.network) + if args.network == 'googlenet': + init_model = tvnet(pretrained=args.pretrained, aux_logits=False) + else: + init_model = tvnet(pretrained=args.pretrained) + set_tvmodel_head_var(init_model) + else: # other models declared in networks package's init + net = getattr(importlib.import_module(name='networks'), args.network) + # WARNING: fixed to pretrained False for other model (non-torchvision) + init_model = net(pretrained=False) + + # Args -- Continual Learning Approach + from approach.incremental_learning import Inc_Learning_Appr + Appr = getattr(importlib.import_module(name='approach.' + args.approach), 'Appr') + assert issubclass(Appr, Inc_Learning_Appr) + appr_args, extra_args = Appr.extra_parser(extra_args) + print('Approach arguments =') + for arg in np.sort(list(vars(appr_args).keys())): + print('\t' + arg + ':', getattr(appr_args, arg)) + print('=' * 108) + + # Args -- Exemplars Management + from datasets.exemplars_dataset import ExemplarsDataset + Appr_ExemplarsDataset = Appr.exemplars_dataset_class() + if Appr_ExemplarsDataset: + assert issubclass(Appr_ExemplarsDataset, ExemplarsDataset) + appr_exemplars_dataset_args, extra_args = Appr_ExemplarsDataset.extra_parser(extra_args) + print('Exemplars dataset arguments =') + for arg in np.sort(list(vars(appr_exemplars_dataset_args).keys())): + print('\t' + arg + ':', getattr(appr_exemplars_dataset_args, arg)) + print('=' * 108) + else: + appr_exemplars_dataset_args = argparse.Namespace() + + # Args -- GridSearch + if args.gridsearch_tasks > 0: + from gridsearch import GridSearch + gs_args, extra_args = GridSearch.extra_parser(extra_args) + Appr_finetuning = getattr(importlib.import_module(name='approach.finetuning'), 'Appr') + assert issubclass(Appr_finetuning, Inc_Learning_Appr) + GridSearch_ExemplarsDataset = Appr.exemplars_dataset_class() + print('GridSearch arguments =') + for arg in np.sort(list(vars(gs_args).keys())): + print('\t' + arg + ':', getattr(gs_args, arg)) + print('=' * 108) + + assert len(extra_args) == 0, "Unused args: {}".format(' '.join(extra_args)) + #################################################################################################################### + + # Log all arguments + full_exp_name = reduce((lambda x, y: x[0] + y[0]), args.datasets) if len(args.datasets) > 0 else args.datasets[0] + full_exp_name += '_' + args.approach + if args.exp_name is not None: + full_exp_name += '_' + args.exp_name + logger = MultiLogger(args.results_path, full_exp_name, loggers=args.log, save_models=args.save_models) + logger.log_args(argparse.Namespace(**args.__dict__, **appr_args.__dict__, **appr_exemplars_dataset_args.__dict__)) + + # Loaders + utils.seed_everything(seed=args.seed) + trn_loader, val_loader, tst_loader, taskcla = get_loaders(args.datasets, args.num_tasks, args.nc_first_task, + args.batch_size, num_workers=args.num_workers, + pin_memory=args.pin_memory) + # Apply arguments for loaders + if args.use_valid_only: + tst_loader = val_loader + max_task = len(taskcla) if args.stop_at_task == 0 else args.stop_at_task + + # Network and Approach instances + utils.seed_everything(seed=args.seed) + net = LLL_Net(init_model, remove_existing_head=not args.keep_existing_head) + utils.seed_everything(seed=args.seed) + # taking transformations and class indices from first train dataset + first_train_ds = trn_loader[0].dataset + transform, class_indices = first_train_ds.transform, first_train_ds.class_indices + appr_kwargs = {**base_kwargs, **dict(logger=logger, **appr_args.__dict__)} + if Appr_ExemplarsDataset: + appr_kwargs['exemplars_dataset'] = Appr_ExemplarsDataset(transform, class_indices, + **appr_exemplars_dataset_args.__dict__) + utils.seed_everything(seed=args.seed) + appr = Appr(net, device, **appr_kwargs) + + # GridSearch + if args.gridsearch_tasks > 0: + ft_kwargs = {**base_kwargs, **dict(logger=logger, + exemplars_dataset=GridSearch_ExemplarsDataset(transform, class_indices))} + appr_ft = Appr_finetuning(net, device, **ft_kwargs) + gridsearch = GridSearch(appr_ft, args.seed, gs_args.gridsearch_config, gs_args.gridsearch_acc_drop_thr, + gs_args.gridsearch_hparam_decay, gs_args.gridsearch_max_num_searches) + + # Loop tasks + print(taskcla) + acc_taw = np.zeros((max_task, max_task)) + acc_tag = np.zeros((max_task, max_task)) + forg_taw = np.zeros((max_task, max_task)) + forg_tag = np.zeros((max_task, max_task)) + for t, (_, ncla) in enumerate(taskcla): + # Early stop tasks if flag + if t >= max_task: + continue + + print('*' * 108) + print('Task {:2d}'.format(t)) + print('*' * 108) + + # Add head for current task + net.add_head(taskcla[t][1]) + net.to(device) + + # GridSearch + if t < args.gridsearch_tasks: + + # Search for best finetuning learning rate -- Maximal Plasticity Search + print('LR GridSearch') + best_ft_acc, best_ft_lr = gridsearch.search_lr(appr.model, t, trn_loader[t], val_loader[t]) + # Apply to approach + appr.lr = best_ft_lr + gen_params = gridsearch.gs_config.get_params('general') + for k, v in gen_params.items(): + if not isinstance(v, list): + setattr(appr, k, v) + + # Search for best forgetting/intransigence tradeoff -- Stability Decay + print('Trade-off GridSearch') + best_tradeoff, tradeoff_name = gridsearch.search_tradeoff(args.approach, appr, + t, trn_loader[t], val_loader[t], best_ft_acc) + # Apply to approach + if tradeoff_name is not None: + setattr(appr, tradeoff_name, best_tradeoff) + + print('-' * 108) + + # Train + appr.train(t, trn_loader[t], val_loader[t]) + print('-' * 108) + + # Test + for u in range(t + 1): + test_loss, acc_taw[t, u], acc_tag[t, u] = appr.eval(u, tst_loader[u]) + if u < t: + forg_taw[t, u] = acc_taw[:t, u].max(0) - acc_taw[t, u] + forg_tag[t, u] = acc_tag[:t, u].max(0) - acc_tag[t, u] + print('>>> Test on task {:2d} : loss={:.3f} | TAw acc={:5.1f}%, forg={:5.1f}%' + '| TAg acc={:5.1f}%, forg={:5.1f}% <<<'.format(u, test_loss, + 100 * acc_taw[t, u], 100 * forg_taw[t, u], + 100 * acc_tag[t, u], 100 * forg_tag[t, u])) + logger.log_scalar(task=t, iter=u, name='loss', group='test', value=test_loss) + logger.log_scalar(task=t, iter=u, name='acc_taw', group='test', value=100 * acc_taw[t, u]) + logger.log_scalar(task=t, iter=u, name='acc_tag', group='test', value=100 * acc_tag[t, u]) + logger.log_scalar(task=t, iter=u, name='forg_taw', group='test', value=100 * forg_taw[t, u]) + logger.log_scalar(task=t, iter=u, name='forg_tag', group='test', value=100 * forg_tag[t, u]) + + # Save + print('Save at ' + os.path.join(args.results_path, full_exp_name)) + logger.log_result(acc_taw, name="acc_taw", step=t) + logger.log_result(acc_tag, name="acc_tag", step=t) + logger.log_result(forg_taw, name="forg_taw", step=t) + logger.log_result(forg_tag, name="forg_tag", step=t) + logger.save_model(net.state_dict(), task=t) + logger.log_result(acc_taw.sum(1) / np.tril(np.ones(acc_taw.shape[0])).sum(1), name="avg_accs_taw", step=t) + logger.log_result(acc_tag.sum(1) / np.tril(np.ones(acc_tag.shape[0])).sum(1), name="avg_accs_tag", step=t) + aux = np.tril(np.repeat([[tdata[1] for tdata in taskcla[:max_task]]], max_task, axis=0)) + logger.log_result((acc_taw * aux).sum(1) / aux.sum(1), name="wavg_accs_taw", step=t) + logger.log_result((acc_tag * aux).sum(1) / aux.sum(1), name="wavg_accs_tag", step=t) + + # Last layer analysis + if args.last_layer_analysis: + weights, biases = last_layer_analysis(net.heads, t, taskcla, y_lim=True) + logger.log_figure(name='weights', iter=t, figure=weights) + logger.log_figure(name='bias', iter=t, figure=biases) + + # Output sorted weights and biases + weights, biases = last_layer_analysis(net.heads, t, taskcla, y_lim=True, sort_weights=True) + logger.log_figure(name='weights', iter=t, figure=weights) + logger.log_figure(name='bias', iter=t, figure=biases) + # Print Summary + utils.print_summary(acc_taw, acc_tag, forg_taw, forg_tag) + print('[Elapsed time = {:.1f} h]'.format((time.time() - tstart) / (60 * 60))) + print('Done!') + + return acc_taw, acc_tag, forg_taw, forg_tag, logger.exp_path + #################################################################################################################### + + +if __name__ == '__main__': + main() diff --git a/src/networks/README.md b/src/networks/README.md new file mode 100644 index 0000000..5832dca --- /dev/null +++ b/src/networks/README.md @@ -0,0 +1,58 @@ +# Networks +We include a core [network](network.py) class which handles the architecture used as well as the heads needed to do +image classification in an incremental learning setting. + +## Main usage +When running an experiment, the network model used can be defined in [main_incremental.py](../main_incremental.py) using +`--network`. By default, the existing head of the architecture (usually with 1,000 outputs because of ImageNet) will be +removed since we create a head each time a task is learned. This default behaviour can be disabled by using +`--keep-existing-head`. If the architecture used has the option to use a pretrained model, it can be called with +`--pretrained`. + +We define a [network](network.py) class which contains the architecture model class to be used (torchvision models or +custom), and also a `ModuleList()` of heads that grows incrementally as tasks are learned, called `model` and `heads` +respectively. When doing a forward pass, inputs are passed through the `model` specific forward pass, and the outputs of +it are then fed to the `heads`. This results in a list of outputs, corresponding to the different tasks learned so far +(multi-head base). However, it has to be noted that when using approaches for class-incremental learning, which has no +access to task-ID during test, the heads are treated as if they were concatenated, so the task-ID has no influence. + +We use this system since it would be equivalent to having a head that grows at each task and it would concatenate the +heads after (or create a new head with the new number of outputs and copy the previous heads weights to their respective +positions). However, an advantage of this system is that it allows to update previous heads by adding them to the +optimizer (needed when using exemplars). This is also important when using some regularization such as weight decay, +which would affect previous task heads/outputs if the corresponding weights are included in the optimizer. Furthermore, +it makes it very easy to evaluate on both task-incremental and class-incremental scenarios. + +### Torchvision models +* Alexnet: `alexnet` +* DenseNet: `densenet121, densenet169, densenet201, densenet161` +* Googlenet: `googlenet` +* Inception: `inception_v3` +* MobileNet: `mobilenet_v2` +* ResNet: `resnet18`, `resnet34`, `resnet50`, `resnet101`, `resnet152`, `resnext50_32x4d`, `resnext101_32x8d` +* ShuffleNet: `shufflenet_v2_x0_5`, `shufflenet_v2_x1_0`, `shufflenet_v2_x1_5`, `shufflenet_v2_x2_0` +* Squeezenet: `squeezenet1_0`, `squeezenet1_1` +* VGG: `vgg11`, `vgg11_bn`, `vgg13`, `vgg13_bn`, `vgg16`, `vgg16_bn`, `vgg19_bn`, `vgg19` +* WideResNet: `wide_resnet50_2`, `wide_resnet101_2` + +### Custom models +We include versions of [LeNet](lenet.py), [ResNet-32](resnet32.py) and [VGGnet](vggnet.py), which use a smaller input +size than the torchvision models. LeNet together with MNIST is useful for quick tests and debugging. + +## Adding new networks +To add a new custom model architecture, follow this: + +1. Take as an example [vggnet.py](vggnet.py) and define a Class for the architecture. Initialize all necessary layers + and modules and define a non-incremental last layer (e.g. `self.fc`). Then add `self.head_var = 'fc'` to point to the + variable containing the non-incremental head (it is not important how many classes it has as output since we remove + it when using it for incremental learning). +2. Define the forward pass of the architecture inside the Class and any other necessary functions. +3. Define a function outside of the Class to call the model. It needs to contain `num_out` and `pretrained` as inputs. +4. Include the import to [\_\_init\_\_.py](__init__.py) and add the architecture name to `allmodels`. + +## Notes +* We provide an implementation of ResNet-32 (see [resnet32.py](resnet32.py)) which is commonly used by several works in + the literature for learning CIFAR-100 in incremental learning scenarios. This network architecture is an adaptation of + ResNet for smaller input size. The number of blocks can be modified in + [this line](https://github.com/mmasana/IL_Survey/blob/9837386d9efddf48d22fc4d23e031248decce68d/src/networks/resnet32.py#L113) + by changing `n=5` to `n=3` for ResNet-20, and `n=9` for ResNet-56. diff --git a/src/networks/__init__.py b/src/networks/__init__.py new file mode 100644 index 0000000..7bb4c78 --- /dev/null +++ b/src/networks/__init__.py @@ -0,0 +1,43 @@ +from torchvision import models + +from .lenet import LeNet +from .vggnet import VggNet +from .resnet32 import resnet32 + +# available torchvision models +tvmodels = ['alexnet', + 'densenet121', 'densenet169', 'densenet201', 'densenet161', + 'googlenet', + 'inception_v3', + 'mobilenet_v2', + 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'shufflenet_v2_x0_5', 'shufflenet_v2_x1_0', 'shufflenet_v2_x1_5', 'shufflenet_v2_x2_0', + 'squeezenet1_0', 'squeezenet1_1', + 'vgg11', 'vgg11_bn', 'vgg13', 'vgg13_bn', 'vgg16', 'vgg16_bn', 'vgg19_bn', 'vgg19', + 'wide_resnet50_2', 'wide_resnet101_2' + ] + +allmodels = tvmodels + ['resnet32', 'LeNet', 'VggNet'] + + +def set_tvmodel_head_var(model): + if type(model) == models.AlexNet: + model.head_var = 'classifier' + elif type(model) == models.DenseNet: + model.head_var = 'classifier' + elif type(model) == models.Inception3: + model.head_var = 'fc' + elif type(model) == models.ResNet: + model.head_var = 'fc' + elif type(model) == models.VGG: + model.head_var = 'classifier' + elif type(model) == models.GoogLeNet: + model.head_var = 'fc' + elif type(model) == models.MobileNetV2: + model.head_var = 'classifier' + elif type(model) == models.ShuffleNetV2: + model.head_var = 'fc' + elif type(model) == models.SqueezeNet: + model.head_var = 'classifier' + else: + raise ModuleNotFoundError diff --git a/src/networks/lenet.py b/src/networks/lenet.py new file mode 100644 index 0000000..2b4f012 --- /dev/null +++ b/src/networks/lenet.py @@ -0,0 +1,30 @@ +from torch import nn +import torch.nn.functional as F + + +class LeNet(nn.Module): + """LeNet-like network for tests with MNIST (28x28).""" + + def __init__(self, in_channels=1, num_classes=10, **kwargs): + super().__init__() + # main part of the network + self.conv1 = nn.Conv2d(in_channels, 6, 5) + self.conv2 = nn.Conv2d(6, 16, 5) + self.fc1 = nn.Linear(16 * 16, 120) + self.fc2 = nn.Linear(120, 84) + + # last classifier layer (head) with as many outputs as classes + self.fc = nn.Linear(84, num_classes) + # and `head_var` with the name of the head, so it can be removed when doing incremental learning experiments + self.head_var = 'fc' + + def forward(self, x): + out = F.relu(self.conv1(x)) + out = F.max_pool2d(out, 2) + out = F.relu(self.conv2(out)) + out = F.max_pool2d(out, 2) + out = out.view(out.size(0), -1) + out = F.relu(self.fc1(out)) + out = F.relu(self.fc2(out)) + out = self.fc(out) + return out diff --git a/src/networks/network.py b/src/networks/network.py new file mode 100644 index 0000000..47095ad --- /dev/null +++ b/src/networks/network.py @@ -0,0 +1,95 @@ +import torch +from torch import nn +from copy import deepcopy + + +class LLL_Net(nn.Module): + """Basic class for implementing networks""" + + def __init__(self, model, remove_existing_head=False): + head_var = model.head_var + assert type(head_var) == str + assert not remove_existing_head or hasattr(model, head_var), \ + "Given model does not have a variable called {}".format(head_var) + assert not remove_existing_head or type(getattr(model, head_var)) in [nn.Sequential, nn.Linear], \ + "Given model's head {} does is not an instance of nn.Sequential or nn.Linear".format(head_var) + super(LLL_Net, self).__init__() + + self.model = model + last_layer = getattr(self.model, head_var) + + if remove_existing_head: + if type(last_layer) == nn.Sequential: + self.out_size = last_layer[-1].in_features + # strips off last linear layer of classifier + del last_layer[-1] + elif type(last_layer) == nn.Linear: + self.out_size = last_layer.in_features + # converts last layer into identity + # setattr(self.model, head_var, nn.Identity()) + # WARNING: this is for when pytorch version is <1.2 + setattr(self.model, head_var, nn.Sequential()) + else: + self.out_size = last_layer.out_features + + self.heads = nn.ModuleList() + self.task_cls = [] + self.task_offset = [] + self._initialize_weights() + + def add_head(self, num_outputs): + """Add a new head with the corresponding number of outputs. Also update the number of classes per task and the + corresponding offsets + """ + self.heads.append(nn.Linear(self.out_size, num_outputs)) + # we re-compute instead of append in case an approach makes changes to the heads + self.task_cls = torch.tensor([head.out_features for head in self.heads]) + self.task_offset = torch.cat([torch.LongTensor(1).zero_(), self.task_cls.cumsum(0)[:-1]]) + + def forward(self, x, return_features=False): + """Applies the forward pass + + Simplification to work on multi-head only -- returns all head outputs in a list + Args: + x (tensor): input images + return_features (bool): return the representations before the heads + """ + x = self.model(x) + assert (len(self.heads) > 0), "Cannot access any head" + y = [] + for head in self.heads: + y.append(head(x)) + if return_features: + return y, x + else: + return y + + def get_copy(self): + """Get weights from the model""" + return deepcopy(self.state_dict()) + + def set_state_dict(self, state_dict): + """Load weights into the model""" + self.load_state_dict(deepcopy(state_dict)) + return + + def freeze_all(self): + """Freeze all parameters from the model, including the heads""" + for param in self.parameters(): + param.requires_grad = False + + def freeze_backbone(self): + """Freeze all parameters from the main model, but not the heads""" + for param in self.model.parameters(): + param.requires_grad = False + + def freeze_bn(self): + """Freeze all Batch Normalization layers from the model and use them in eval() mode""" + for m in self.model.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + + def _initialize_weights(self): + """Initialize weights using different strategies""" + # TODO: add different initialization strategies + pass diff --git a/src/networks/resnet32.py b/src/networks/resnet32.py new file mode 100644 index 0000000..cddec4a --- /dev/null +++ b/src/networks/resnet32.py @@ -0,0 +1,115 @@ +import torch.nn as nn + +__all__ = ['resnet32'] + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + out = self.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + if self.downsample is not None: + residual = self.downsample(x) + out += residual + return self.relu(out) + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + if self.downsample is not None: + residual = self.downsample(x) + out += residual + return self.relu(out) + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=10): + self.inplanes = 16 + super(ResNet, self).__init__() + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.relu = nn.ReLU(inplace=True) + self.layer1 = self._make_layer(block, 16, layers[0]) + self.layer2 = self._make_layer(block, 32, layers[1], stride=2) + self.layer3 = self._make_layer(block, 64, layers[2], stride=2) + self.avgpool = nn.AvgPool2d(8, stride=1) + # last classifier layer (head) with as many outputs as classes + self.fc = nn.Linear(64 * block.expansion, num_classes) + # and `head_var` with the name of the head, so it can be removed when doing incremental learning experiments + self.head_var = 'fc' + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.relu(self.bn1(self.conv1(x))) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.avgpool(x) + x = x.view(x.size(0), -1) + x = self.fc(x) + return x + + +def resnet32(pretrained=False, **kwargs): + if pretrained: + raise NotImplementedError + # change n=3 for ResNet-20, and n=9 for ResNet-56 + n = 5 + model = ResNet(BasicBlock, [n, n, n], **kwargs) + return model diff --git a/src/networks/vggnet.py b/src/networks/vggnet.py new file mode 100644 index 0000000..5af8da3 --- /dev/null +++ b/src/networks/vggnet.py @@ -0,0 +1,58 @@ +from torch import nn +import torch.nn.functional as F + + +class VggNet(nn.Module): + """ Following the VGGnet based on VGG16 but for smaller input (64x64) + Check this blog for some info: https://learningai.io/projects/2017/06/29/tiny-imagenet.html + """ + + def __init__(self, num_classes=1000): + super().__init__() + + self.features = nn.Sequential( + nn.Conv2d(3, 64, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(64, 64, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.Conv2d(64, 128, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(128, 128, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.Conv2d(128, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.Conv2d(256, 512, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(512, 512, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=2, stride=2), + ) + self.fc6 = nn.Linear(in_features=512 * 4 * 4, out_features=4096, bias=True) + self.fc7 = nn.Linear(in_features=4096, out_features=4096, bias=True) + # last classifier layer (head) with as many outputs as classes + self.fc = nn.Linear(in_features=4096, out_features=num_classes, bias=True) + # and `head_var` with the name of the head, so it can be removed when doing incremental learning experiments + self.head_var = 'fc' + + def forward(self, x): + h = self.features(x) + h = h.view(x.size(0), -1) + h = F.dropout(F.relu(self.fc6(h))) + h = F.dropout(F.relu(self.fc7(h))) + h = self.fc(h) + return h + + +def vggnet(num_out=100, pretrained=False): + if pretrained: + raise NotImplementedError + return VggNet(num_out) diff --git a/src/tests/README.md b/src/tests/README.md new file mode 100644 index 0000000..9bd5efa --- /dev/null +++ b/src/tests/README.md @@ -0,0 +1,33 @@ +# Tests +The tests in this folder are our tool to check if everything is working as intended and to pick any errors that might +appear when introducing new features. They can also be used to make sure all dependencies are available before running +any long experiments. + +## Running tests + +### From console +Type the following code in your console: +```bash +cd src/ +py.test -s tests/ +``` + +### Running tests in parallel +As the amount of tests grow, it can be faster to run them in parallel since they can take few minutes each. +Pytest can support it by using the `pytest-xdist` plugin, e.g. running all tests with 5 workers: +```bash +cd src/ +py.test -n 5 tests/ +``` +And for a more verbose output: +```bash +cd src/ +py.test -sv -n 5 tests/ +``` +**_Warning:_** It is recommended to run a single test without parallelization the first time, since the first thing that +our test will do is download the dataset (MNIST). If ran in parallel they can start downloading it in multiple workers +at the same time. + +### From your IDE (PyCharm, VSCode, ...) +`py.tests` are well supported. It's usually enough to select `py.test` as a framework, right click on a test file or +directory and select the option to run pytest tests (not as a python script!). diff --git a/src/tests/__init__.py b/src/tests/__init__.py new file mode 100644 index 0000000..c24786c --- /dev/null +++ b/src/tests/__init__.py @@ -0,0 +1,52 @@ +import os +import torch +import shutil + +from main_incremental import main +import datasets.dataset_config as c + + +def run_main(args_line, result_dir='results_test', clean_run=False): + assert "--results-path" not in args_line + + print('Staring dir:', os.getcwd()) + if os.getcwd().endswith('tests'): + os.chdir('..') + elif os.getcwd().endswith('IL_Survey'): + os.chdir('src') + elif os.getcwd().endswith('src'): + print('CWD is OK.') + print('Test CWD:', os.getcwd()) + test_results_path = os.getcwd() + f"/../{result_dir}" + + # for testing - use relative path to CWD + c.dataset_config['mnist']['path'] = '../data' + if os.path.exists(test_results_path) and clean_run: + shutil.rmtree(test_results_path) + os.makedirs(test_results_path, exist_ok=True) + args_line += " --results-path {}".format(test_results_path) + + # if distributed test -- use all GPU + worker_id = int(os.environ.get("PYTEST_XDIST_WORKER", "gw-1")[2:]) + if worker_id >= 0 and torch.cuda.is_available(): + gpu_idx = worker_id % torch.cuda.device_count() + args_line += " --gpu {}".format(gpu_idx) + + print('ARGS:', args_line) + return main(args_line.split(' ')) + + +def run_main_and_assert(args_line, + taw_current_task_min=0.01, + tag_current_task_min=0.0, + result_dir='results_test'): + acc_taw, acc_tag, forg_taw, forg_tag, exp_dir = run_main(args_line, result_dir) + + # acc matrices sanity check + assert acc_tag.shape == acc_taw.shape + assert acc_tag.shape == forg_tag.shape + assert acc_tag.shape == forg_taw.shape + + # check current task performance + assert all(acc_tag.diagonal() >= tag_current_task_min) + assert all(acc_taw.diagonal() >= taw_current_task_min) diff --git a/src/tests/test_bic.py b/src/tests/test_bic.py new file mode 100644 index 0000000..d300e8c --- /dev/null +++ b/src/tests/test_bic.py @@ -0,0 +1,34 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach bic" + + +def test_bic_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_bic_exemplars_lambda(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --lamb 1" + run_main_and_assert(args_line) + + +def test_bic_exemplars_per_class(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + run_main_and_assert(args_line) + + +def test_bic_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_dataloader.py b/src/tests/test_dataloader.py new file mode 100644 index 0000000..ed170e1 --- /dev/null +++ b/src/tests/test_dataloader.py @@ -0,0 +1,42 @@ +import torch +from torch.utils.data import DataLoader +from torch.utils.data.dataset import TensorDataset + +from main_incremental import main + + +def test_dataloader_dataset_swap(): + # given + data1 = TensorDataset(torch.arange(10)) + data2 = TensorDataset(torch.arange(10, 20)) + dl = DataLoader(data1, batch_size=2, shuffle=True, num_workers=1) + # when + batches1 = list(dl) + + try: + dl.dataset += data2 + except ValueError: + # In new pytorch this raise an error + # which is expected and OK behaviour + return + + batches2 = list(dl) + all_data = list(dl.dataset) + + # then + assert len(all_data) == 20 + assert len(batches1) == 5 + assert len(batches2) == 5 + # ^ is troublesome! + # Sampler is initialized in DataLoader __init__ + # and it holding reference to old DS. + assert dl.sampler.data_source == data1 + # Thus, we will not see the new data. + + +def test_dataloader_multiple_datasets(): + args_line = "--exp-name local_test --approach finetuning --datasets mnist mnist mnist" \ + " --network LeNet --num-tasks 2 --batch-size 32" \ + " --results-path ../results/ --num-workers 0 --nepochs 2" + print('ARGS:', args_line) + main(args_line.split(' ')) diff --git a/src/tests/test_datasets_transforms.py b/src/tests/test_datasets_transforms.py new file mode 100644 index 0000000..c0f5909 --- /dev/null +++ b/src/tests/test_datasets_transforms.py @@ -0,0 +1,35 @@ +import numpy as np +from torchvision.transforms import Lambda +from torch.utils.data.dataset import ConcatDataset + +from datasets.memory_dataset import MemoryDataset +from datasets.exemplars_selection import override_dataset_transform + + +def pic(i): + return np.array([[i]], dtype=np.int8) + + +def test_dataset_transform_override(): + # given + data1 = MemoryDataset({ + 'x': [pic(1), pic(2), pic(3)], 'y': ['a', 'b', 'c'] + }, transform=Lambda(lambda x: np.array(x)[0, 0] * 2)) + data2 = MemoryDataset({ + 'x': [pic(4), pic(5), pic(6)], 'y': ['d', 'e', 'f'] + }, transform=Lambda(lambda x: np.array(x)[0, 0] * 3)) + data3 = MemoryDataset({ + 'x': [pic(7), pic(8), pic(9)], 'y': ['g', 'h', 'i'] + }, transform=Lambda(lambda x: np.array(x)[0, 0] + 10)) + ds = ConcatDataset([data1, ConcatDataset([data2, data3])]) + + # when + x1, y1 = zip(*[ds[i] for i in range(len(ds))]) + with override_dataset_transform(ds, Lambda(lambda x: np.array(x)[0, 0])) as ds_overriden: + x2, y2 = zip(*[ds_overriden[i] for i in range(len(ds_overriden))]) + x3, y3 = zip(*[ds[i] for i in range(len(ds))]) + + # then + assert np.array_equal(x1, [2, 4, 6, 12, 15, 18, 17, 18, 19]) + assert np.array_equal(x2, [1, 2, 3, 4, 5, 6, 7, 8, 9]) + assert np.array_equal(x3, x1) # after everything is back to normal diff --git a/src/tests/test_dmc.py b/src/tests/test_dmc.py new file mode 100644 index 0000000..45ba3c1 --- /dev/null +++ b/src/tests/test_dmc.py @@ -0,0 +1,13 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets cifar100" \ + " --network resnet32 --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach dmc" \ + " --aux-dataset cifar100" + + +def test_dmc(): + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + diff --git a/src/tests/test_eeil.py b/src/tests/test_eeil.py new file mode 100644 index 0000000..91eb16a --- /dev/null +++ b/src/tests/test_eeil.py @@ -0,0 +1,31 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach eeil" + + +def test_eeil_exemplars_with_noise_grad(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --nepochs-finetuning 1" + args_line += " --noise-grad" + run_main_and_assert(args_line) + + +def test_eeil_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --nepochs-finetuning 1" + run_main_and_assert(args_line) + + +def test_eeil_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + args_line += " --nepochs-finetuning 1" + run_main_and_assert(args_line) diff --git a/src/tests/test_ewc.py b/src/tests/test_ewc.py new file mode 100644 index 0000000..6bdda73 --- /dev/null +++ b/src/tests/test_ewc.py @@ -0,0 +1,25 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach ewc" + + +def test_ewc_without_exemplars(): + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + + +def test_ewc_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_ewc_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_finetuning.py b/src/tests/test_finetuning.py new file mode 100644 index 0000000..5712022 --- /dev/null +++ b/src/tests/test_finetuning.py @@ -0,0 +1,78 @@ +import torch +import pytest + +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0" + + +def test_finetuning_without_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + run_main_and_assert(args_line) + + +def test_finetuning_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +@pytest.mark.xfail +def test_finetuning_with_exemplars_per_class_and_herding(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars-per-class 10" + args_line += " --exemplar-selection herding" + run_main_and_assert(args_line) + + +def test_finetuning_with_exemplars_per_class_and_entropy(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars-per-class 10" + args_line += " --exemplar-selection entropy" + run_main_and_assert(args_line) + + +def test_finetuning_with_exemplars_per_class_and_distance(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars-per-class 10" + args_line += " --exemplar-selection distance" + run_main_and_assert(args_line) + + +def test_wrong_args(): + with pytest.raises(SystemExit): # error of providing both args + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars-per-class 10" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_finetuning_with_eval_on_train(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars-per-class 10" + args_line += " --exemplar-selection distance" + args_line += " --eval-on-train" + run_main_and_assert(args_line) + +def test_finetuning_with_no_cudnn_deterministic(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --num-exemplars-per-class 10" + args_line += " --exemplar-selection distance" + + run_main_and_assert(args_line) + assert torch.backends.cudnn.deterministic == True + + args_line += " --no-cudnn-deterministic" + run_main_and_assert(args_line) + assert torch.backends.cudnn.deterministic == False diff --git a/src/tests/test_fix_bn.py b/src/tests/test_fix_bn.py new file mode 100644 index 0000000..2e8e3ee --- /dev/null +++ b/src/tests/test_fix_bn.py @@ -0,0 +1,80 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0 --fix-bn" + + +def test_finetuning_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + run_main_and_assert(args_line) + + +def test_joint_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach joint" + run_main_and_assert(args_line) + + +def test_freezingt_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach freezing" + run_main_and_assert(args_line) + + +def test_icarl_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --approach icarl" + run_main_and_assert(args_line) + + +def test_eeil_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --approach eeil" + run_main_and_assert(args_line) + + +def test_mas_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach mas" + run_main_and_assert(args_line) + + +def test_lwf_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach lwf" + run_main_and_assert(args_line) + + +def test_lwm_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach lwm --gradcam-layer conv2 --log-gradcam-samples 16" + run_main_and_assert(args_line) + + +def test_r_walk_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach r_walk" + run_main_and_assert(args_line) + + +def test_path_integral_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach path_integral" + run_main_and_assert(args_line) + + +def test_luci_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach lucir" + run_main_and_assert(args_line) + + +def test_ewc_fix_bn(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach ewc" + run_main_and_assert(args_line) diff --git a/src/tests/test_freezing.py b/src/tests/test_freezing.py new file mode 100644 index 0000000..ad2e14e --- /dev/null +++ b/src/tests/test_freezing.py @@ -0,0 +1,24 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach freezing" + + +def test_freezing_without_exemplars(): + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + + +def test_freezing_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_freezing_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + run_main_and_assert(args_line) diff --git a/src/tests/test_gridsearch.py b/src/tests/test_gridsearch.py new file mode 100644 index 0000000..216540d --- /dev/null +++ b/src/tests/test_gridsearch.py @@ -0,0 +1,92 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3 --num-workers 0" \ + " --gridsearch-tasks 3 --gridsearch-config gridsearch_config" \ + " --gridsearch-acc-drop-thr 0.2 --gridsearch-hparam-decay 0.5" + + +def test_gridsearch_finetuning(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_freezing(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach freezing --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_joint(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach joint" + run_main_and_assert(args_line) + + +def test_gridsearch_lwf(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach lwf --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_icarl(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach icarl --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_eeil(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach eeil --nepochs-finetuning 3 --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_bic(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach bic --num-bias-epochs 3 --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_lucir(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach lucir --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_lwm(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach lwm --gradcam-layer conv2 --log-gradcam-samples 16 --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_ewc(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach ewc --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_mas(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach mas --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_pathint(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach path_integral --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_rwalk(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach r_walk --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_gridsearch_dmc(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach dmc" + args_line += " --aux-dataset mnist" # just to test the grid search fast + run_main_and_assert(args_line) diff --git a/src/tests/test_icarl.py b/src/tests/test_icarl.py new file mode 100644 index 0000000..a13074c --- /dev/null +++ b/src/tests/test_icarl.py @@ -0,0 +1,30 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach icarl" + + +def test_icarl_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --lamb 1" + run_main_and_assert(args_line) + + +def test_icarl_exemplars_without_lamb(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + args_line += " --lamb 0" + run_main_and_assert(args_line) + + +def test_icarl_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + args_line += " --lamb 1" + run_main_and_assert(args_line) diff --git a/src/tests/test_il2m.py b/src/tests/test_il2m.py new file mode 100644 index 0000000..97c1a67 --- /dev/null +++ b/src/tests/test_il2m.py @@ -0,0 +1,21 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach il2m" + + +def test_il2m(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_il2m_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_joint.py b/src/tests/test_joint.py new file mode 100644 index 0000000..83ae18b --- /dev/null +++ b/src/tests/test_joint.py @@ -0,0 +1,12 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0" + + +def test_joint(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach joint" + run_main_and_assert(args_line) diff --git a/src/tests/test_last_layer_analysis.py b/src/tests/test_last_layer_analysis.py new file mode 100644 index 0000000..6290199 --- /dev/null +++ b/src/tests/test_last_layer_analysis.py @@ -0,0 +1,14 @@ +from tests import run_main + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0" \ + " --approach finetuning" + + +def test_last_layer_analysis(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --last-layer-analysis" + run_main(args_line) + diff --git a/src/tests/test_loggers.py b/src/tests/test_loggers.py new file mode 100644 index 0000000..5fb71e7 --- /dev/null +++ b/src/tests/test_loggers.py @@ -0,0 +1,31 @@ +import pandas as pd +from pathlib import Path + +from tests import run_main + +FAST_LOCAL_TEST_ARGS = "--exp-name loggers_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0 --approach finetuning" + + +def test_disk_and_tensorflow_logger(): + + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --log disk tensorboard" + result = run_main(args_line, 'results_test_loggers', clean_run=True) + experiment_dir = Path(result[-1]) + + # check disk logger + assert experiment_dir.is_dir() + raw_logs = list(experiment_dir.glob('raw_log-*.txt')) + assert len(raw_logs) == 1 + df = pd.read_json(raw_logs[0], lines=True) + assert sorted(df.iter.unique()) == [0, 1, 2] + assert sorted(df.group.unique()) == ['test', 'train', 'valid'] + assert len(df.group.unique()) == 3 + + # check tb logger + tb_events_logs = list(experiment_dir.glob('events.out.tfevents*')) + assert len(tb_events_logs) == 1 + assert experiment_dir.joinpath(tb_events_logs[0]).is_file() diff --git a/src/tests/test_lucir.py b/src/tests/test_lucir.py new file mode 100644 index 0000000..ad90e68 --- /dev/null +++ b/src/tests/test_lucir.py @@ -0,0 +1,49 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --gridsearch-tasks -1" \ + " --approach lucir" + + +def test_lucir_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + run_main_and_assert(args_line) + + +def test_lucir_exemplars_with_gridsearch(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + args_line = args_line.replace('--gridsearch-tasks -1', '--gridsearch-tasks 3') + run_main_and_assert(args_line) + + +def test_lucir_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + run_main_and_assert(args_line) + + +def test_lucir_exemplars_remove_margin_ranking(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + args_line += " --remove-margin-ranking" + run_main_and_assert(args_line) + + +def test_lucir_exemplars_remove_adapt_lamda(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + args_line += " --remove-adapt-lamda" + run_main_and_assert(args_line) + + +def test_lucir_exemplars_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars-per-class 20" + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + run_main_and_assert(args_line) diff --git a/src/tests/test_lwf.py b/src/tests/test_lwf.py new file mode 100644 index 0000000..8e8a8e9 --- /dev/null +++ b/src/tests/test_lwf.py @@ -0,0 +1,25 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach lwf" + + +def test_lwf_without_exemplars(): + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + + +def test_lwf_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_lwf_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_lwm.py b/src/tests/test_lwm.py new file mode 100644 index 0000000..0b2deb8 --- /dev/null +++ b/src/tests/test_lwm.py @@ -0,0 +1,27 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach lwm --gradcam-layer conv2" + + +def test_lwm_without_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --log-gradcam-samples 16" + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + + +def test_lwm_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_lwm_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_mas.py b/src/tests/test_mas.py new file mode 100644 index 0000000..fc36d42 --- /dev/null +++ b/src/tests/test_mas.py @@ -0,0 +1,25 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach mas" + + +def test_mas_without_exemplars(): + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + + +def test_mas_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_mas_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_multisoftmax.py b/src/tests/test_multisoftmax.py new file mode 100644 index 0000000..d6bb783 --- /dev/null +++ b/src/tests/test_multisoftmax.py @@ -0,0 +1,19 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0" + + +def test_finetuning_without_multisoftmax(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + run_main_and_assert(args_line) + + +def test_finetuning_with_multisoftmax(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --multi-softmax" + run_main_and_assert(args_line) diff --git a/src/tests/test_path_integral.py b/src/tests/test_path_integral.py new file mode 100644 index 0000000..ea0b987 --- /dev/null +++ b/src/tests/test_path_integral.py @@ -0,0 +1,25 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach path_integral" + + +def test_pi_without_exemplars(): + run_main_and_assert(FAST_LOCAL_TEST_ARGS) + + +def test_pi_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_pi_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_rwalk.py b/src/tests/test_rwalk.py new file mode 100644 index 0000000..8efcf74 --- /dev/null +++ b/src/tests/test_rwalk.py @@ -0,0 +1,27 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 3" \ + " --num-workers 0" \ + " --approach r_walk" + + +def test_rwalk_without_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 0" + run_main_and_assert(args_line) + + +def test_rwalk_with_exemplars(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) + + +def test_rwalk_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + args_line += " --num-exemplars 200" + run_main_and_assert(args_line) diff --git a/src/tests/test_stop_at_task.py b/src/tests/test_stop_at_task.py new file mode 100644 index 0000000..5358824 --- /dev/null +++ b/src/tests/test_stop_at_task.py @@ -0,0 +1,11 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 5 --seed 1 --batch-size 32" \ + " --nepochs 2 --num-workers 0 --stop-at-task 3" + + +def test_finetuning_stop_at_task(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + run_main_and_assert(args_line) diff --git a/src/tests/test_warmup.py b/src/tests/test_warmup.py new file mode 100644 index 0000000..72a1231 --- /dev/null +++ b/src/tests/test_warmup.py @@ -0,0 +1,20 @@ +from tests import run_main_and_assert + +FAST_LOCAL_TEST_ARGS = "--exp-name local_test --datasets mnist" \ + " --network LeNet --num-tasks 3 --seed 1 --batch-size 32" \ + " --nepochs 2 --lr-factor 10 --momentum 0.9 --lr-min 1e-7" \ + " --num-workers 0" + + +def test_finetuning_without_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + run_main_and_assert(args_line) + + +def test_finetuning_with_warmup(): + args_line = FAST_LOCAL_TEST_ARGS + args_line += " --approach finetuning" + args_line += " --warmup-nepochs 5" + args_line += " --warmup-lr-factor 0.5" + run_main_and_assert(args_line) diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..b728417 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,34 @@ +import os +import torch +import random +import numpy as np + +cudnn_deterministic = True + + +def seed_everything(seed=0): + """Fix all random seeds""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + torch.backends.cudnn.deterministic = cudnn_deterministic + + +def print_summary(acc_taw, acc_tag, forg_taw, forg_tag): + """Print summary of results""" + for name, metric in zip(['TAw Acc', 'TAg Acc', 'TAw Forg', 'TAg Forg'], [acc_taw, acc_tag, forg_taw, forg_tag]): + print('*' * 108) + print(name) + for i in range(metric.shape[0]): + print('\t', end='') + for j in range(metric.shape[1]): + print('{:5.1f}% '.format(100 * metric[i, j]), end='') + if np.trace(metric) == 0.0: + if i > 0: + print('\tAvg.:{:5.1f}% '.format(100 * metric[i, :i].mean()), end='') + else: + print('\tAvg.:{:5.1f}% '.format(100 * metric[i, :i + 1].mean()), end='') + print() + print('*' * 108)