From 596d421ddd79af8e5e73b00eb81d71ff9a6d6aed Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 17:04:49 +0000 Subject: [PATCH 01/11] made compatible with tensorboard --- examples/example.py | 42 +++++++++++++-------- mlogger/config.py | 49 +++++++++++++++++++----- mlogger/container.py | 24 ++++++++++-- mlogger/metric/__init__.py | 73 ++++++++++++++++++++++++++++-------- mlogger/metric/base.py | 77 ++++++++++++++++++++++++++++++++------ 5 files changed, 209 insertions(+), 56 deletions(-) diff --git a/examples/example.py b/examples/example.py index cc071d1..5cb3895 100644 --- a/examples/example.py +++ b/examples/example.py @@ -4,6 +4,7 @@ import numpy as np from builtins import range +from torch.utils.tensorboard import SummaryWriter np.random.seed(1234) @@ -43,6 +44,7 @@ def oracle(data, target): # some hyper-parameters of the experiment use_visdom = True +use_tensorboard = True lr = 0.01 n_epochs = 10 @@ -52,32 +54,37 @@ def oracle(data, target): # log the hyperparameters of the experiment if use_visdom: - plotter = mlogger.VisdomPlotter({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, + visdom_plotter = mlogger.VisdomPlotter({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, manual_update=True) else: - plotter = None + visdom_plotter = None + +if use_tensorboard: + summary_writer = SummaryWriter() +else: + summary_writer = None xp = mlogger.Container() -xp.config = mlogger.Config(plotter=plotter) +xp.config = mlogger.Config(visdom_plotter=visdom_plotter, summary_writer=summary_writer) xp.config.update(lr=lr, n_epochs=n_epochs) xp.epoch = mlogger.metric.Simple() xp.train = mlogger.Container() -xp.train.acc1 = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@1", plot_legend="training") -xp.train.acck = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@k", plot_legend="training") -xp.train.loss = mlogger.metric.Average(plotter=plotter, plot_title="Objective") -xp.train.timer = mlogger.metric.Timer(plotter=plotter, plot_title="Time", plot_legend="training") +xp.train.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@1", plot_legend="training") +xp.train.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@k", plot_legend="training") +xp.train.loss = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Objective") +xp.train.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Time", plot_legend="training") xp.val = mlogger.Container() -xp.val.acc1 = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@1", plot_legend="validation") -xp.val.acck = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@k", plot_legend="validation") -xp.val.timer = mlogger.metric.Timer(plotter=plotter, plot_title="Time", plot_legend="validation") +xp.val.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@1", plot_legend="validation") +xp.val.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@k", plot_legend="validation") +xp.val.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Time", plot_legend="validation") xp.val_best = mlogger.Container() -xp.val_best.acc1 = mlogger.metric.Maximum(plotter=plotter, plot_title="Accuracy@1", plot_legend="validation-best") -xp.val_best.acck = mlogger.metric.Maximum(plotter=plotter, plot_title="Accuracy@k", plot_legend="validation-best") +xp.val_best.acc1 = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@1", plot_legend="validation-best") +xp.val_best.acck = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@k", plot_legend="validation-best") #---------------------------------------------------------- @@ -129,7 +136,7 @@ def oracle(data, target): print("Prec@1: \t {0:.2f}%".format(xp.val_best.acc1.value)) print("Prec@k: \t {0:.2f}%".format(xp.val_best.acck.value)) -plotter.update_plots() +visdom_plotter.update_plots() #---------------------------------------------------------- # Save & load experiment @@ -141,12 +148,15 @@ def oracle(data, target): xp.save_to('state.json') -new_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, +new_visdom_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, manual_update=True) +new_summary_writer = SummaryWriter() new_xp = mlogger.load_container('state.json') -new_xp.plot_on(new_plotter) -new_plotter.update_plots() +new_xp.plot_on_visdom(new_visdom_plotter) +new_visdom_plotter.update_plots() + +# new_xp.plot_on_tensorboard(new_summary_writer) print('Current train loss value: {}'.format(new_xp.train.loss.value)) new_xp.train.loss.update(2) diff --git a/mlogger/config.py b/mlogger/config.py index f6871a2..e34bc57 100644 --- a/mlogger/config.py +++ b/mlogger/config.py @@ -5,19 +5,29 @@ import time import sys import mlogger +import warnings class Config(object): - def __init__(self, plotter=None, plot_title=None, - get_general_info=True, get_git_info=False, **kwargs): + def __init__(self, plotter=None, plot_title=None, get_general_info=True, + get_git_info=False, visdom_plotter=None, summary_writer=None, **kwargs): object.__setattr__(self, '_state', {}) if plotter is not None: - self.plot_on(plotter, plot_title) - else: - object.__setattr__(self, '_plotter', plotter) - object.__setattr__(self, '_plot_title', plot_title) + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + visdom_plotter = plotter + del plotter + + object.__setattr__(self, '_visdom_plotter', visdom_plotter) + object.__setattr__(self, '_summary_writer', summary_writer) + object.__setattr__(self, '_plot_title', plot_title) + + if visdom_plotter is not None: + self.plot_on_visdom(visdom_plotter, plot_title) + + if summary_writer is not None: + self.plot_on_tensorboard(summary_writer) if get_general_info: self.update_general_info() @@ -55,8 +65,10 @@ def state_dict(self): def update(self, **kwargs): self._state.update(kwargs) - if self._plotter is not None: - self._plotter._update_text(self._plot_title, kwargs) + if self._visdom_plotter is not None: + self._visdom_plotter._update_text(self._plot_title, kwargs) + if self._summary_writer is not None: + self._summary_writer.add_hparams(kwargs, {}) return self def load_state_dict(self, state): @@ -69,16 +81,33 @@ def __repr__(self): return repr_ def plot_on(self, plotter, plot_title): + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + return self.plot_on_visdom(plotter, plot_title) + + def plot_on_visdom(self, visdom_plotter, plot_title): # plot current state if len(self._state): - plotter._update_text(plot_title, self._state) + visdom_plotter._update_text(plot_title, self._state) # store for future logs - object.__setattr__(self, '_plotter', plotter) + object.__setattr__(self, '_visdom_plotter', visdom_plotter) object.__setattr__(self, '_plot_title', plot_title) return self + def plot_on_tensorboard(self, summary_writer, plot_title=None): + if plot_title: + warnings.warn("warning argument ignored", RuntimeWarning) + + # plot current state + if len(self._state): + summary_writer.add_hparams(self._state, {}) + + # store for future logs + object.__setattr__(self, '_summary_writer', summary_writer) + + return self + def __getattr__(self, key): if key == '_state': return object.__getattr__(self, _state) diff --git a/mlogger/container.py b/mlogger/container.py index b8fae33..8252c93 100644 --- a/mlogger/container.py +++ b/mlogger/container.py @@ -4,6 +4,7 @@ import numpy as np from builtins import dict from collections import defaultdict, OrderedDict +import warnings class Container(object): @@ -73,17 +74,34 @@ def named_metrics(self): return named_metrics_list def plot_on(self, plotter): + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + self.plot_on_visdom(plotter) + + def plot_on_visdom(self, visdom_plotter): for child in self.children(): if isinstance(child, Container): - child.plot_on(plotter) + child.plot_on_visdom(visdom_plotter) elif isinstance(child, mlogger.Config): plot_title = child._plot_title - child.plot_on(plotter, plot_title) + child.plot_on_visdom(visdom_plotter, plot_title) + elif isinstance(child, mlogger.metric.Base): + plot_title = child._plot_title + plot_legend = child._plot_legend + if plot_title is not None: + child.plot_on_visdom(visdom_plotter, plot_title, plot_legend) + + + def plot_on_tensorboard(self, summary_writer): + for child in self.children(): + if isinstance(child, Container): + child.plot_on_tensorboard(summary_writer) + elif isinstance(child, mlogger.Config): + child.plot_on_tensorboard(summary_writer) elif isinstance(child, mlogger.metric.Base): plot_title = child._plot_title plot_legend = child._plot_legend if plot_title is not None: - child.plot_on(plotter, plot_title, plot_legend) + child.plot_on_tensorboard(summary_writer, plot_title, plot_legend) def __repr__(self): _repr = "Container()" diff --git a/mlogger/metric/__init__.py b/mlogger/metric/__init__.py index 5375c3d..e3352b3 100644 --- a/mlogger/metric/__init__.py +++ b/mlogger/metric/__init__.py @@ -1,5 +1,6 @@ import numpy as np import time +import warnings from collections import defaultdict, OrderedDict from future.utils import viewitems @@ -14,8 +15,13 @@ class Simple(Base): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): - super(Simple, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + import warnings + super(Simple, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._val = 0. @@ -44,8 +50,13 @@ def __repr__(self): class TNT(Base): - def __init__(self, tnt_meter, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): - super(TNT, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, tnt_meter, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + import warnings + super(TNT, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) self._tnt_meter = tnt_meter def reset(self): @@ -70,8 +81,13 @@ def __repr__(self): class Timer(Base): - def __init__(self, plotter=None, plot_title=None, plot_legend=None): - super(Timer, self).__init__(False, plotter, plot_title, plot_legend) + def __init__(self, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + super(Timer, self).__init__(time_indexing=False, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self.start = time.time() @@ -105,8 +121,13 @@ def __repr__(self): class Maximum(Base): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): - super(Maximum, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + super(Maximum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._val = -np.inf @@ -143,8 +164,13 @@ def __repr__(self): class Minimum(Base): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): - super(Minimum, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + super(Minimum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._val = np.inf @@ -184,8 +210,13 @@ class Accumulator_(Base): """ Credits to the authors of pytorch/tnt for this. """ - def __init__(self, time_indexing, plotter=None, plot_title=None, plot_legend=None): - super(Accumulator_, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, time_indexing, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + super(Accumulator_, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._avg = 0 @@ -213,8 +244,13 @@ def value(self): class Average(Accumulator_): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): - super(Average, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + super(Average, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) @property def value(self): @@ -230,8 +266,13 @@ def __repr__(self): class Sum(Accumulator_): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): - super(Sum, self).__init__(time_indexing, plotter, plot_title, plot_legend) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): + if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + super(Sum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) @property def value(self): diff --git a/mlogger/metric/base.py b/mlogger/metric/base.py index 14d22fb..0d8fa95 100644 --- a/mlogger/metric/base.py +++ b/mlogger/metric/base.py @@ -1,5 +1,6 @@ import numpy as np import time +import warnings from collections import defaultdict, OrderedDict from future.utils import viewitems @@ -8,8 +9,11 @@ from .to_float import to_float + + class Base(object): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None): + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, + visdom_plotter=None, summary_writer=None): """ Basic metric """ @@ -21,12 +25,23 @@ def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legen self.reset_hooks_on_log() if plotter is not None: + assert visdom_plotter is None + visdom_plotter = plotter + del plotter + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + + self._summary_writer = summary_writer + self._visdom_plotter = visdom_plotter + self._plot_title = plot_title + self._plot_legend = plot_legend + + if summary_writer is not None: assert plot_title is not None, "a plot title is required" - self.plot_on(plotter, plot_title, plot_legend) - else: - self._plotter = plotter - self._plot_title = plot_title - self._plot_legend = plot_legend + self.plot_on_tensorboard(summary_writer, plot_title, plot_legend) + + if visdom_plotter is not None: + assert plot_title is not None, "a plot title is required" + self.plot_on_visdom(visdom_plotter, plot_title, plot_legend) def init_history(self, time_indexing): self._history = History(time_indexing) @@ -76,9 +91,23 @@ def log(self, time=None): self._history.log(event_time, value) - # plot current value - if self._plotter is not None: - self._plotter._update_xy(title=self._plot_title, legend=self._plot_legend, x=event_time, y=value) + # plot current value on visdom + if self._visdom_plotter is not None: + self._visdom_plotter._update_xy(title=self._plot_title, legend=self._plot_legend, x=event_time, y=value) + + # plot current value on tensorboard + if self._summary_writer is not None: + if self._plot_legend: + tag = "{title}/{legend}".format(title=self._plot_title, legend=self._plot_legend) + else: + tag = self._plot_title + + if self._time_indexing: + opts = dict(global_step=len(self._history._times), walltime=event_time) + else: + opts = dict(global_step=event_time) + + self._summary_writer.add_scalar(tag=tag, scalar_value=value, **opts) for hook in self.hooks_on_log: hook() @@ -104,14 +133,40 @@ def last_logged(self): return self._history._last_value def plot_on(self, plotter, plot_title, plot_legend=None): + warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + self.plot_on_visdom(plotter, plot_title, plot_legend) + + def plot_on_visdom(self, visdom_plotter, plot_title, plot_legend=None): # plot current state x, y = self._history._times, self._history._values assert len(x) == len(y) if len(x): - plotter._update_xy(plot_title, plot_legend, x, y) + visdom_plotter._update_xy(plot_title, plot_legend, x, y) + + # store for future logs + self._visdom_plotter = visdom_plotter + self._plot_title = plot_title + self._plot_legend = plot_legend + + return self + + def plot_on_tensorboard(self, summary_writer, plot_title, plot_legend=None): + # plot current state + x, y = self._history._times, self._history._values + assert len(x) == len(y) + if plot_legend: + tag = "{title}/{legend}".format(title=plot_title, legend=plot_legend) + else: + tag = plot_title + for i, (x_scalar, y_scalar) in enumerate(zip(x, y)): + if self._time_indexing: + opts = dict(global_step=i, walltime=x_scalar) + else: + opts = dict(global_step=x_scalar) + summary_writer.add_scalar(tag=tag, scalar_value=y_scalar, **opts) # store for future logs - self._plotter = plotter + self._summary_writer = summary_writer self._plot_title = plot_title self._plot_legend = plot_legend From cb06dd6e458a6ce76d3ae1f88aa1458fa3872540 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 17:09:57 +0000 Subject: [PATCH 02/11] cleaned up example --- examples/example.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/examples/example.py b/examples/example.py index 5cb3895..618839b 100644 --- a/examples/example.py +++ b/examples/example.py @@ -66,7 +66,7 @@ def oracle(data, target): xp = mlogger.Container() -xp.config = mlogger.Config(visdom_plotter=visdom_plotter, summary_writer=summary_writer) +xp.config = mlogger.Config(plotter=visdom_plotter, summary_writer=summary_writer) xp.config.update(lr=lr, n_epochs=n_epochs) xp.epoch = mlogger.metric.Simple() @@ -136,7 +136,8 @@ def oracle(data, target): print("Prec@1: \t {0:.2f}%".format(xp.val_best.acc1.value)) print("Prec@k: \t {0:.2f}%".format(xp.val_best.acck.value)) -visdom_plotter.update_plots() +if use_visdom: + visdom_plotter.update_plots() #---------------------------------------------------------- # Save & load experiment @@ -148,15 +149,18 @@ def oracle(data, target): xp.save_to('state.json') -new_visdom_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, - manual_update=True) -new_summary_writer = SummaryWriter() + new_xp = mlogger.load_container('state.json') -new_xp.plot_on_visdom(new_visdom_plotter) -new_visdom_plotter.update_plots() -# new_xp.plot_on_tensorboard(new_summary_writer) +if use_visdom: + new_visdom_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, + manual_update=True) + new_xp.plot_on_visdom(new_visdom_plotter) + new_visdom_plotter.update_plots() +if use_tensorboard: + new_summary_writer = SummaryWriter() + new_xp.plot_on_tensorboard(new_summary_writer) print('Current train loss value: {}'.format(new_xp.train.loss.value)) new_xp.train.loss.update(2) From f97d25ef7e351360119f1a082d7dd3b232909f1b Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 19:27:16 +0000 Subject: [PATCH 03/11] added FutureWarnings and tidied up code --- mlogger/config.py | 6 +- mlogger/container.py | 3 +- mlogger/metric/__init__.py | 109 ++++++++++++++++++------------------- mlogger/metric/base.py | 8 +-- mlogger/stdout.py | 6 +- 5 files changed, 64 insertions(+), 68 deletions(-) diff --git a/mlogger/config.py b/mlogger/config.py index e34bc57..84467b8 100644 --- a/mlogger/config.py +++ b/mlogger/config.py @@ -9,13 +9,13 @@ class Config(object): - def __init__(self, plotter=None, plot_title=None, get_general_info=True, + def __init__(self, plotter=None, plot_title=None, get_general_info=True, get_git_info=False, visdom_plotter=None, summary_writer=None, **kwargs): object.__setattr__(self, '_state', {}) if plotter is not None: - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) visdom_plotter = plotter del plotter @@ -81,7 +81,7 @@ def __repr__(self): return repr_ def plot_on(self, plotter, plot_title): - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) return self.plot_on_visdom(plotter, plot_title) def plot_on_visdom(self, visdom_plotter, plot_title): diff --git a/mlogger/container.py b/mlogger/container.py index 8252c93..eb717ce 100644 --- a/mlogger/container.py +++ b/mlogger/container.py @@ -74,7 +74,7 @@ def named_metrics(self): return named_metrics_list def plot_on(self, plotter): - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) self.plot_on_visdom(plotter) def plot_on_visdom(self, visdom_plotter): @@ -90,7 +90,6 @@ def plot_on_visdom(self, visdom_plotter): if plot_title is not None: child.plot_on_visdom(visdom_plotter, plot_title, plot_legend) - def plot_on_tensorboard(self, summary_writer): for child in self.children(): if isinstance(child, Container): diff --git a/mlogger/metric/__init__.py b/mlogger/metric/__init__.py index e3352b3..e89a348 100644 --- a/mlogger/metric/__init__.py +++ b/mlogger/metric/__init__.py @@ -14,14 +14,22 @@ ] +def _deprecate_plotter_argument(plotter, visdom_plotter): + if plotter is not None: + if visdom_plotter is None or visdom_plotter == plotter: + warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) + else: + raise ValueError("Arguments 'plotter', and 'visdom_plotter', are different and both not None") + + return visdom_plotter + + class Simple(Base): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - import warnings - super(Simple, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, + visdom_plotter=None, summary_writer=None): + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + super(Simple, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._val = 0. @@ -50,13 +58,13 @@ def __repr__(self): class TNT(Base): - def __init__(self, tnt_meter, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - import warnings - super(TNT, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, tnt_meter, time_indexing=None, plotter=None, plot_title=None, + plot_legend=None, visdom_plotter=None, summary_writer=None): + + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + + super(TNT, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) self._tnt_meter = tnt_meter def reset(self): @@ -82,12 +90,11 @@ def __repr__(self): class Timer(Base): def __init__(self, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) - super(Timer, self).__init__(time_indexing=False, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + + super(Timer, self).__init__(time_indexing=False, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self.start = time.time() @@ -121,13 +128,12 @@ def __repr__(self): class Maximum(Base): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) - super(Maximum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, + visdom_plotter=None, summary_writer=None): + + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + super(Maximum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._val = -np.inf @@ -164,13 +170,12 @@ def __repr__(self): class Minimum(Base): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) - super(Minimum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, + visdom_plotter=None, summary_writer=None): + + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + super(Minimum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._val = np.inf @@ -210,13 +215,11 @@ class Accumulator_(Base): """ Credits to the authors of pytorch/tnt for this. """ - def __init__(self, time_indexing, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) - super(Accumulator_, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, time_indexing, plotter=None, plot_title=None, plot_legend=None, + visdom_plotter=None, summary_writer=None): + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + super(Accumulator_, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) def reset(self): self._avg = 0 @@ -244,13 +247,11 @@ def value(self): class Average(Accumulator_): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) - super(Average, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, + visdom_plotter=None, summary_writer=None): + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + super(Average, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) @property def value(self): @@ -266,13 +267,11 @@ def __repr__(self): class Sum(Accumulator_): - def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): - if plotter is not None: - assert visdom_plotter is None - visdom_plotter = plotter - del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) - super(Sum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, visdom_plotter=visdom_plotter, summary_writer=summary_writer) + def __init__(self, time_indexing=None, plotter=None, plot_title=None, + plot_legend=None, visdom_plotter=None, summary_writer=None): + visdom_plotter = _deprecate_plotter_argument(plotter, visdom_plotter) + super(Sum, self).__init__(time_indexing=time_indexing, plot_title=plot_title, plot_legend=plot_legend, + visdom_plotter=visdom_plotter, summary_writer=summary_writer) @property def value(self): diff --git a/mlogger/metric/base.py b/mlogger/metric/base.py index 0d8fa95..9a88f0a 100644 --- a/mlogger/metric/base.py +++ b/mlogger/metric/base.py @@ -9,8 +9,6 @@ from .to_float import to_float - - class Base(object): def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legend=None, visdom_plotter=None, summary_writer=None): @@ -28,7 +26,7 @@ def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legen assert visdom_plotter is None visdom_plotter = plotter del plotter - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) self._summary_writer = summary_writer self._visdom_plotter = visdom_plotter @@ -41,7 +39,7 @@ def __init__(self, time_indexing=None, plotter=None, plot_title=None, plot_legen if visdom_plotter is not None: assert plot_title is not None, "a plot title is required" - self.plot_on_visdom(visdom_plotter, plot_title, plot_legend) + self.plot_on_visdom(visdom_plotter, plot_title, plot_legend) def init_history(self, time_indexing): self._history = History(time_indexing) @@ -133,7 +131,7 @@ def last_logged(self): return self._history._last_value def plot_on(self, plotter, plot_title, plot_legend=None): - warnings.warn("use visdom_plotter instead of plotter", FutureWarning) + warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) self.plot_on_visdom(plotter, plot_title, plot_legend) def plot_on_visdom(self, visdom_plotter, plot_title, plot_legend=None): diff --git a/mlogger/stdout.py b/mlogger/stdout.py index 893cbbf..64555f5 100644 --- a/mlogger/stdout.py +++ b/mlogger/stdout.py @@ -29,9 +29,9 @@ def write(self, message): self.log.write(message) def flush(self): - #this flush method is needed for python 3 compatibility. - #this handles the flush command by doing nothing. - #you might want to specify some extra behavior here. + # this flush method is needed for python 3 compatibility. + # this handles the flush command by doing nothing. + # you might want to specify some extra behavior here. pass From 2d1867e026def4f0f5fa07060968c59cdfd2c4a7 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 19:41:31 +0000 Subject: [PATCH 04/11] tidied up example.py, fixed warning in __init__.py --- examples/example.py | 46 ++++++++++++++++++++++---------------- mlogger/metric/__init__.py | 3 ++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/examples/example.py b/examples/example.py index 618839b..83334e1 100644 --- a/examples/example.py +++ b/examples/example.py @@ -48,14 +48,14 @@ def oracle(data, target): lr = 0.01 n_epochs = 10 -#---------------------------------------------------------- +# ---------------------------------------------------------- # Prepare logging -#---------------------------------------------------------- +# ---------------------------------------------------------- # log the hyperparameters of the experiment if use_visdom: visdom_plotter = mlogger.VisdomPlotter({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, - manual_update=True) + manual_update=True) else: visdom_plotter = None @@ -72,24 +72,33 @@ def oracle(data, target): xp.epoch = mlogger.metric.Simple() xp.train = mlogger.Container() -xp.train.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@1", plot_legend="training") -xp.train.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@k", plot_legend="training") -xp.train.loss = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Objective") -xp.train.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Time", plot_legend="training") +xp.train.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@1", plot_legend="training") +xp.train.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@k", plot_legend="training") +xp.train.loss = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Objective") +xp.train.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Time", plot_legend="training") xp.val = mlogger.Container() -xp.val.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@1", plot_legend="validation") -xp.val.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@k", plot_legend="validation") -xp.val.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Time", plot_legend="validation") +xp.val.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@1", plot_legend="validation") +xp.val.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@k", plot_legend="validation") +xp.val.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Time", plot_legend="validation") xp.val_best = mlogger.Container() -xp.val_best.acc1 = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@1", plot_legend="validation-best") -xp.val_best.acck = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, plot_title="Accuracy@k", plot_legend="validation-best") +xp.val_best.acc1 = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@1", plot_legend="validation-best") +xp.val_best.acck = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@k", plot_legend="validation-best") -#---------------------------------------------------------- +# ---------------------------------------------------------- # Training -#---------------------------------------------------------- +# ---------------------------------------------------------- for epoch in range(n_epochs): @@ -139,9 +148,9 @@ def oracle(data, target): if use_visdom: visdom_plotter.update_plots() -#---------------------------------------------------------- +# ---------------------------------------------------------- # Save & load experiment -#---------------------------------------------------------- +# ---------------------------------------------------------- xp.train.loss.reset() xp.train.loss.update(1) @@ -150,12 +159,11 @@ def oracle(data, target): xp.save_to('state.json') - new_xp = mlogger.load_container('state.json') if use_visdom: - new_visdom_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, - manual_update=True) + new_visdom_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', + 'port': 8097}, manual_update=True) new_xp.plot_on_visdom(new_visdom_plotter) new_visdom_plotter.update_plots() if use_tensorboard: diff --git a/mlogger/metric/__init__.py b/mlogger/metric/__init__.py index e89a348..0508217 100644 --- a/mlogger/metric/__init__.py +++ b/mlogger/metric/__init__.py @@ -16,10 +16,11 @@ def _deprecate_plotter_argument(plotter, visdom_plotter): if plotter is not None: - if visdom_plotter is None or visdom_plotter == plotter: + if (visdom_plotter is None) or (visdom_plotter is plotter): warnings.warn("Argument `plotter` is deprecated. Please use `visdom_plotter` instead.", FutureWarning) else: raise ValueError("Arguments 'plotter', and 'visdom_plotter', are different and both not None") + visdom_plotter = plotter return visdom_plotter From f8d741999d7b55ec2312296acbba612711746339 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 19:48:30 +0000 Subject: [PATCH 05/11] tiny fix in visdom_plotter.py --- mlogger/plotter/visdom_plotter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlogger/plotter/visdom_plotter.py b/mlogger/plotter/visdom_plotter.py index 5e83a53..8ce4b5b 100644 --- a/mlogger/plotter/visdom_plotter.py +++ b/mlogger/plotter/visdom_plotter.py @@ -80,5 +80,6 @@ def load_state_dict(self, state): self.text_wins[title].load_state_dict(text_win_state) def __repr__(self): - repr_ = "VisdomPlotter({visdom_opts}, {manual_update})".format(visdom_opts=self.visdom_opts, manual_update=self.manual_update) + repr_ = "VisdomPlotter({visdom_opts}, {manual_update})".format(visdom_opts=self.visdom_opts, + manual_update=self.manual_update) return repr_ From e50eea32469c5f8071ecc34b40eaf58a4d10a045 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 20:00:43 +0000 Subject: [PATCH 06/11] small change to example.py (use visdom_plotter rather than plotter) --- examples/example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example.py b/examples/example.py index 83334e1..6a284c2 100644 --- a/examples/example.py +++ b/examples/example.py @@ -66,7 +66,7 @@ def oracle(data, target): xp = mlogger.Container() -xp.config = mlogger.Config(plotter=visdom_plotter, summary_writer=summary_writer) +xp.config = mlogger.Config(visdom_plotter=visdom_plotter, summary_writer=summary_writer) xp.config.update(lr=lr, n_epochs=n_epochs) xp.epoch = mlogger.metric.Simple() From 6a7f50b0fed7315cce2ba3867e07fb25dff77297 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 20:17:01 +0000 Subject: [PATCH 07/11] updated readme file --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 4c93b4b..8cb3dbc 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ print(new_xp.total_timer.value) # 0.0001 * Improve your user experience with `visdom`: * Ease of use: ```python - plotter = mlogger.VisdomPlotter(({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}) - acc = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy") + visdom_plotter = mlogger.VisdomPlotter(({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}) + acc = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy") acc.update(100) acc.update(92) print(acc.value) # 96.0 @@ -83,32 +83,32 @@ n_epochs = 10 # log the hyperparameters of the experiment if use_visdom: - plotter = mlogger.VisdomPlotter({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, + visdom_plotter = mlogger.VisdomPlotter({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, manual_update=True) else: - plotter = None + visdom_plotter = None xp = mlogger.Container() -xp.config = mlogger.Config(plotter=plotter) +xp.config = mlogger.Config(visdom_plotter=visdom_plotter) xp.config.update(lr=lr, n_epochs=n_epochs) xp.epoch = mlogger.metric.Simple() xp.train = mlogger.Container() -xp.train.acc1 = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@1", plot_legend="training") -xp.train.acck = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@k", plot_legend="training") -xp.train.loss = mlogger.metric.Average(plotter=plotter, plot_title="Objective") -xp.train.timer = mlogger.metric.Timer(plotter=plotter, plot_title="Time", plot_legend="training") +xp.train.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@1", plot_legend="training") +xp.train.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@k", plot_legend="training") +xp.train.loss = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Objective") +xp.train.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, plot_title="Time", plot_legend="training") xp.val = mlogger.Container() -xp.val.acc1 = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@1", plot_legend="validation") -xp.val.acck = mlogger.metric.Average(plotter=plotter, plot_title="Accuracy@k", plot_legend="validation") -xp.val.timer = mlogger.metric.Timer(plotter=plotter, plot_title="Time", plot_legend="validation") +xp.val.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@1", plot_legend="validation") +xp.val.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@k", plot_legend="validation") +xp.val.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, plot_title="Time", plot_legend="validation") xp.val_best = mlogger.Container() -xp.val_best.acc1 = mlogger.metric.Maximum(plotter=plotter, plot_title="Accuracy@1", plot_legend="validation-best") -xp.val_best.acck = mlogger.metric.Maximum(plotter=plotter, plot_title="Accuracy@k", plot_legend="validation-best") +xp.val_best.acc1 = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, plot_title="Accuracy@1", plot_legend="validation-best") +xp.val_best.acck = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, plot_title="Accuracy@k", plot_legend="validation-best") #---------------------------------------------------------- @@ -160,7 +160,7 @@ print("-" * 50) print("Prec@1: \t {0:.2f}%".format(xp.val_best.acc1.value)) print("Prec@k: \t {0:.2f}%".format(xp.val_best.acck.value)) -plotter.update_plots() +visdom_plotter.update_plots() #---------------------------------------------------------- # Save & load experiment @@ -176,7 +176,7 @@ new_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server manual_update=True) new_xp = mlogger.load_container('state.json') -new_xp.plot_on(new_plotter) +new_xp.plot_on_visdom(new_plotter) new_plotter.update_plots() print('Current train loss value: {}'.format(new_xp.train.loss.value)) From 5cca3deae731f80f7fa23016ce3854d16eece945 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 20:23:15 +0000 Subject: [PATCH 08/11] updated readme - copied example code --- README.md | 65 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8cb3dbc..bf28e9e 100644 --- a/README.md +++ b/README.md @@ -74,46 +74,61 @@ import numpy as np # some hyper-parameters of the experiment use_visdom = True +use_tensorboard = True lr = 0.01 n_epochs = 10 -#---------------------------------------------------------- +# ---------------------------------------------------------- # Prepare logging -#---------------------------------------------------------- +# ---------------------------------------------------------- # log the hyperparameters of the experiment if use_visdom: visdom_plotter = mlogger.VisdomPlotter({'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, - manual_update=True) + manual_update=True) else: visdom_plotter = None +if use_tensorboard: + summary_writer = SummaryWriter() +else: + summary_writer = None + xp = mlogger.Container() -xp.config = mlogger.Config(visdom_plotter=visdom_plotter) +xp.config = mlogger.Config(visdom_plotter=visdom_plotter, summary_writer=summary_writer) xp.config.update(lr=lr, n_epochs=n_epochs) xp.epoch = mlogger.metric.Simple() xp.train = mlogger.Container() -xp.train.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@1", plot_legend="training") -xp.train.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@k", plot_legend="training") -xp.train.loss = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Objective") -xp.train.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, plot_title="Time", plot_legend="training") +xp.train.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@1", plot_legend="training") +xp.train.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@k", plot_legend="training") +xp.train.loss = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Objective") +xp.train.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Time", plot_legend="training") xp.val = mlogger.Container() -xp.val.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@1", plot_legend="validation") -xp.val.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, plot_title="Accuracy@k", plot_legend="validation") -xp.val.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, plot_title="Time", plot_legend="validation") +xp.val.acc1 = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@1", plot_legend="validation") +xp.val.acck = mlogger.metric.Average(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@k", plot_legend="validation") +xp.val.timer = mlogger.metric.Timer(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Time", plot_legend="validation") xp.val_best = mlogger.Container() -xp.val_best.acc1 = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, plot_title="Accuracy@1", plot_legend="validation-best") -xp.val_best.acck = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, plot_title="Accuracy@k", plot_legend="validation-best") +xp.val_best.acc1 = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@1", plot_legend="validation-best") +xp.val_best.acck = mlogger.metric.Maximum(visdom_plotter=visdom_plotter, summary_writer=summary_writer, + plot_title="Accuracy@k", plot_legend="validation-best") -#---------------------------------------------------------- +# ---------------------------------------------------------- # Training -#---------------------------------------------------------- +# ---------------------------------------------------------- for epoch in range(n_epochs): @@ -160,11 +175,12 @@ print("-" * 50) print("Prec@1: \t {0:.2f}%".format(xp.val_best.acc1.value)) print("Prec@k: \t {0:.2f}%".format(xp.val_best.acck.value)) -visdom_plotter.update_plots() +if use_visdom: + visdom_plotter.update_plots() -#---------------------------------------------------------- +# ---------------------------------------------------------- # Save & load experiment -#---------------------------------------------------------- +# ---------------------------------------------------------- xp.train.loss.reset() xp.train.loss.update(1) @@ -172,12 +188,17 @@ print('Train loss value before saving state: {}'.format(xp.train.loss.value)) xp.save_to('state.json') -new_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', 'port': 8097}, - manual_update=True) new_xp = mlogger.load_container('state.json') -new_xp.plot_on_visdom(new_plotter) -new_plotter.update_plots() + +if use_visdom: + new_visdom_plotter = mlogger.VisdomPlotter(visdom_opts={'env': 'my_experiment', 'server': 'http://localhost', + 'port': 8097}, manual_update=True) + new_xp.plot_on_visdom(new_visdom_plotter) + new_visdom_plotter.update_plots() +if use_tensorboard: + new_summary_writer = SummaryWriter() + new_xp.plot_on_tensorboard(new_summary_writer) print('Current train loss value: {}'.format(new_xp.train.loss.value)) new_xp.train.loss.update(2) From 695622480495e535019922dd99aeffc85593353f Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 20:49:29 +0000 Subject: [PATCH 09/11] further changes to readme --- README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bf28e9e..b0cb3b9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ # MLogger: a Machine Learning logger -Currently in version alpha, the API might undergo some minor changes. ## Installation @@ -49,6 +48,21 @@ print(new_xp.total_timer.value) # 0.0001 * Robustness: if `visdom` fails to send data (due to a network instability for instance), `logger` automatically caches it and tries to send it together with the next request * Performance: you can manually choose when to update the `visdom` plots. This permits to batch the data being sent and yields considerable speedups when logging thousands or more points per second. + +* Improve your user experience with `tensorboard`: + * Ease of use: + ```python + summary_writer = SummaryWriter() + acc = mlogger.metric.Average(summary_writer=summary_writer, plot_title="Accuracy") + acc.update(100) + acc.update(92) + print(acc.value) # 96.0 + acc.log() # automatically sends 96.0 to visdom server on window with title 'Accuracy' + ``` + * TODO + * Performance: you can manually choose when to update the `visdom` plots. This permits to batch the data being sent and yields considerable speedups when logging thousands or more points per second. + + * Save all output printed in the console to a text file ```python with mlogger.stdout_to('printed_stuff.txt'): @@ -67,9 +81,9 @@ The following example shows some functionalities of the package (full example co import mlogger import numpy as np -#... +# ... # code to generate fake data -#... +# ... # some hyper-parameters of the experiment From e1830aa44b9791478b4e08d4e67884c842492a67 Mon Sep 17 00:00:00 2001 From: FlorianJaeckle Date: Wed, 4 Dec 2019 20:59:24 +0000 Subject: [PATCH 10/11] updated version, small changes to readme --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b0cb3b9..e83c7df 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ print(new_xp.total_timer.value) # 0.0001 acc.log() # automatically sends 96.0 to visdom server on window with title 'Accuracy' ``` * TODO - * Performance: you can manually choose when to update the `visdom` plots. This permits to batch the data being sent and yields considerable speedups when logging thousands or more points per second. + * Performance: you can manually choose when to update the `tensorboard` plots. This permits to batch the data being sent and yields considerable speedups when logging thousands or more points per second. * Save all output printed in the console to a text file diff --git a/setup.py b/setup.py index 559d76f..97add20 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -__version__ = "1.0.1a" +__version__ = "1.1.0" setup(name='mlogger', description='A Machine Learning logger', From 6630f72351d05e93190d2532e3c37916858e1090 Mon Sep 17 00:00:00 2001 From: lberrada Date: Mon, 9 Dec 2019 11:02:58 +0000 Subject: [PATCH 11/11] README update --- README.md | 27 ++++++++++++--------------- setup.py | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e83c7df..0f45c85 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,18 @@ print(new_xp.train.accuracy.value) # 97.0 print(new_xp.total_timer.value) # 0.0001 ``` +* Easily visualize results on `tensorboard` with pytorch: +```python +from torch.utils.tensorboard import SummaryWriter + +summary_writer = SummaryWriter() +acc = mlogger.metric.Average(summary_writer=summary_writer, plot_title="Accuracy") +acc.update(100) +acc.update(92) +print(acc.value) # 96.0 +acc.log() # automatically logs 96.0 to tensorboard (displayed with title 'Accuracy') +``` + * Improve your user experience with `visdom`: * Ease of use: ```python @@ -48,21 +60,6 @@ print(new_xp.total_timer.value) # 0.0001 * Robustness: if `visdom` fails to send data (due to a network instability for instance), `logger` automatically caches it and tries to send it together with the next request * Performance: you can manually choose when to update the `visdom` plots. This permits to batch the data being sent and yields considerable speedups when logging thousands or more points per second. - -* Improve your user experience with `tensorboard`: - * Ease of use: - ```python - summary_writer = SummaryWriter() - acc = mlogger.metric.Average(summary_writer=summary_writer, plot_title="Accuracy") - acc.update(100) - acc.update(92) - print(acc.value) # 96.0 - acc.log() # automatically sends 96.0 to visdom server on window with title 'Accuracy' - ``` - * TODO - * Performance: you can manually choose when to update the `tensorboard` plots. This permits to batch the data being sent and yields considerable speedups when logging thousands or more points per second. - - * Save all output printed in the console to a text file ```python with mlogger.stdout_to('printed_stuff.txt'): diff --git a/setup.py b/setup.py index 97add20..9302bca 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup(name='mlogger', description='A Machine Learning logger', - author='Leonard Berrada', + author='Leonard Berrada, Florian Jaeckle', packages=find_packages(), license="MIT License", url='https://github.com/oval-group/mlogger',