Adjust default values for batching (#934)

pipeline/translate/decoder.yml

@@ -1,10 +1,16 @@
 normalize: 1.0
 word-penalty: 0
-mini-batch: 16
-mini-batch-words: 500
-maxi-batch: 1000
+
+# TODO #936 - These values can be unsafe for certain machines.
+
+# Batch size tuned for four Tesla V100-SXM2-16GB
+# See: https://github.com/mozilla/translations/issues/931
+mini-batch-words: 5000
+maxi-batch: 10000
 maxi-batch-sort: src
+fp16: true
+
 max-length: 200
 max-length-crop: true
 beam-size: 8
-quiet-translation: True
+quiet-translation: True

taskcluster/configs/config.prod.yml

@@ -216,9 +216,12 @@ marian-args:
     beam-size: '12'
     mini-batch-words: '2000'
   decoding-teacher:
-    mini-batch-words: '4000'
     precision: float16
-
+    # Batch size tuned for four Tesla V100-SXM2-16GB
+    # See: https://github.com/mozilla/translations/issues/931
+    mini-batch-words: '5000'
+    maxi-batch: '10000'
+    fp16: true
   # Early stopping can be adjusted to ensure models converge. See:
   # https://mozilla.github.io/firefox-translations-training/training-guide.html#model-training
   training-backward: