opus1m+bt-2021-05-01.yml

release: mkh-eng/opus1m+bt-2021-05-01.zip
release-date: 2021-05-01
dataset-name: opus1m+bt
modeltype: transformer-align
vocabulary:
   source: opus1m+bt.spm32k-spm32k.vocab.yml
   target: opus1m+bt.spm32k-spm32k.vocab.yml
pre-processing: normalization + SentencePiece (spm32k,spm32k)
subwords:
   source: spm32k
   target: spm32k
subword-models:
   source: source.spm
   target: target.spm
source-languages:
   - kha
   - khm
   - mnw
   - ngt
   - vie
target-languages:
   - eng
training-data:
   khm-eng: Tatoeba-train (289332) 
   khm_Latn-eng: Tatoeba-train (10780) 
   vie-eng: Tatoeba-train (1000000) wikibooks.aa (991044) wikinews.aa (457120) wikipedia.aa (981412) wikipedia.ab (981543) wikipedia.ac (981642) wikipedia.ad (981433) wikiquote.aa (997067) 
validation-data:
   eng-khm: Tatoeba-dev, 965
   eng-khm_Latn: Tatoeba-dev, 35
   eng-vie: Tatoeba-dev, 2824
   total-size-shuffled: 1997
   devset-selected: top 1997  lines of Tatoeba-dev.src.shuffled
test-data:
   Tatoeba-test.kha-eng: 1314/8445
   Tatoeba-test.khm-eng: 752/4394
   Tatoeba-test.khm_Latn-eng: 11/79
   Tatoeba-test.mnw-eng: 9/86
   Tatoeba-test.ngt-eng: 17/100
   Tatoeba-test.multi-eng: 4592/33266
   Tatoeba-test.vie-eng: 2500/20240
   Tatoeba-test.vie_Hani-eng: 1/9
   tico19-test.khm-eng: 2100/56848
BLEU-scores:
   Tatoeba-test.kha-eng: 0.8
   Tatoeba-test.khm-eng: 0.2
   Tatoeba-test.khm_Latn-eng: 0.8
   Tatoeba-test.mnw-eng: 0.3
   Tatoeba-test.ngt-eng: 0.4
   Tatoeba-test.multi-eng: 24.0
   Tatoeba-test.vie-eng: 37.8
   Tatoeba-test.vie_Hani-eng: 0.0
   tico19-test.khm-eng: 1.2
chr-F-scores:
   Tatoeba-test.kha-eng: 0.115
   Tatoeba-test.khm-eng: 0.129
   Tatoeba-test.khm_Latn-eng: 0.072
   Tatoeba-test.mnw-eng: 0.090
   Tatoeba-test.ngt-eng: 0.083
   Tatoeba-test.multi-eng: 0.391
   Tatoeba-test.vie-eng: 0.561
   Tatoeba-test.vie_Hani-eng: 0.042
   tico19-test.khm-eng: 0.190