Skip to content

Commit

Permalink
Only report sampled task counts on master device
Browse files Browse the repository at this point in the history
  • Loading branch information
Waino committed Sep 23, 2024
1 parent d5ae7da commit fe56020
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions mammoth/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,12 +304,17 @@ def train(
# Learning rate used to be retrieved with: self.optim.learning_rate()
# However, as each optimizer has its own learning rate, it is not obvious what to log here.
# We might log the mean or the range of learning rates, but the simplest thing is to log nothing.
report_lr = None
if device_context.is_master():
sampled_task_counts = self.task_queue_manager.sampled_task_counts
else:
sampled_task_counts = None
report_stats = self._maybe_report_training(
step,
train_steps,
None,
report_lr,
report_stats,
sampled_task_counts=self.task_queue_manager.sampled_task_counts,
sampled_task_counts=sampled_task_counts,
)

if step % valid_steps == 0 and valid_iter is not None:
Expand Down

0 comments on commit fe56020

Please sign in to comment.