Rebuild binary dataset alphabet when selecting all alphabet-changing

transformations Fixes #616.
mittagessen · Jun 19, 2024 · d483227 · d483227
1 parent 803553f
commit d483227
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/kraken/lib/train.py b/kraken/lib/train.py
@@ -457,8 +457,10 @@ def _build_dataset(self,
                 dataset.add(**sample)
             except KrakenInputException as e:
                 logger.warning(str(e))
-        if self.format_type == 'binary' and self.hparams.hyper_params['normalization']:
-            logger.debug('Rebuilding dataset using unicode normalization')
+        if self.format_type == 'binary' and (self.hparams.hyper_params['normalization'] or
+                                             self.hparams.hyper_params['normalize_whitespace'] or
+                                             self.reorder):
+            logger.debug('Text transformations modifying alphabet selected. Rebuilding alphabet')
             dataset.rebuild_alphabet()
 
         return dataset