Skip to content

Commit

Permalink
missing counts added
Browse files Browse the repository at this point in the history
  • Loading branch information
@jorgtied
jorgtied committed Nov 10, 2023
1 parent 6faf2da commit f691911
Show file tree
Hide file tree
Showing 7 changed files with 62 additions and 26 deletions.
30 changes: 16 additions & 14 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,15 +127,10 @@ GET_ISO_CODE = ${ISO639} -m
OPUSREAD_ARGS =


## settings for sort (parsort is part of GNU parallel)
## settings for sort

THREADS ?= 4
ifneq (${shell which parsort 2>/dev/null},)
SORT = LC_ALL=C parsort -T ${TMPDIR} -S100M
else
SORT = LC_ALL=C sort -T ${TMPDIR} -S100M --parallel=${THREADS}
endif

SORT := LC_ALL=C sort -T ${TMPDIR} -S100M --parallel=${THREADS}

## tool for shuffling data (terashuf or sort)

Expand All @@ -144,6 +139,12 @@ ifeq (${SHUFFLE},)
SHUFFLE = ${SORT} --random-sort
endif

## check whether parsort is available

ifneq (${shell which parsort 2>/dev/null},)
SORT := LC_ALL=C parsort -T ${TMPDIR} -S100M
endif


## use pigz if available

Expand All @@ -162,13 +163,19 @@ endif
## basic training data filtering pipeline
## TODO: get rid of recode?
## TODO: use GNU parallel?
## NEW: use ftfy library for basic unicode fixing

BASIC_FILTERS = | perl -CS -pe 'tr[\x{9}\x{A}\x{D}\x{20}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}][]cd;' \
| recode -f utf8..utf16 | recode -f utf16..utf8 \
BASIC_FILTERS = | ftfy \
| $(TOKENIZER)/deescape-special-chars.perl \
| sed 's/(Translated with Google Translate)//g'


#BASIC_FILTERS = | perl -CS -pe 'tr[\x{9}\x{A}\x{D}\x{20}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}][]cd;' \
# | recode -f utf8..utf16 | recode -f utf16..utf8 \
# | $(TOKENIZER)/deescape-special-chars.perl \
# | sed 's/(Translated with Google Translate)//g'


## DONE: remove lines 2 and 3 (they do too much, also remove emojis)
##
# | perl -CS -pe 's/\&\s*\#\s*160\s*\;/ /g' \
Expand Down Expand Up @@ -339,11 +346,6 @@ all: opus-langpairs3.txt
${MAKE} released-data-counts
${MAKE} release-tag

fix-release-2023:
${MAKE} dev-tsv test-tsv test-release dev-release
${MAKE} ${DATADIR}/README.md
${MAKE} langids subsets



## TODO: there is some kind of memory-leak somewhere that causes jobs to crahs
Expand Down
6 changes: 6 additions & 0 deletions data/release/v2023-09-26/released-bitexts-highest.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ bel-rus Belarusian Russian 2500 15122 15162 2832 17270 17324 22998266 277245748
ben-eng Bengali English 2500 10942 11321 3063 14225 15462 78764429 669896165 713813703
bul-deu Bulgarian German 317 1639 1827 47383235 617102672 615370548
bul-eng Bulgarian English 10000 56740 60705 7835 44204 47335 113270896 1453244814 1528332996
bul-fra Bulgarian French 451 2724 3243 43704625 475998935 536869706
bul-ita Bulgarian Italian 2500 13344 14074 4006 21635 22681 43149114 548361970 584782283
bul-rus Bulgarian Russian 1247 6556 6155 1001 5184 4806 44461448 458102211 419678955
bul-spa Bulgarian Spanish 286 1359 1460 50263683 518602296 580362453
Expand Down Expand Up @@ -44,10 +45,12 @@ dan-fin Danish Finnish 2665 13452 11009 1854 10314 8348 49411725 778227129 59147
dan-fra Danish French 1731 9307 10245 89 601 690 77220719 1240133470 1433684311
dan-hun Danish Hungarian 882 4967 3888 6 26 18 39096324 568567353 515841359
dan-ita Danish Italian 286 1876 1885 4 16 18 63225160 1077789214 1171882384
dan-nld Danish Dutch 1643 8485 8952 67 338 365 71005999 1160485402 1258693815
dan-nor Danish Norwegian 1311 8343 8216 1050 6607 6561 39503372 541325880 535545627
dan-por Danish Portuguese 876 4281 4368 4 33 24 73045085 1099601319 1219803678
dan-rus Danish Russian 1713 9788 8376 1065 5999 5207 19444845 180706655 164465722
dan-spa Danish Spanish 5000 30181 29832 5165 31534 31087 81687841 1280222017 1487592915
dan-swe Danish Swedish 2348 12542 12292 1440 7859 7730 67424560 1051984859 1026298995
dan-tur Danish Turkish 770 3555 2680 4 16 12 19482641 182994406 150078891
dan-zho Danish Chinese 250 1400 265 12949875 111175424 23597243
deu-ell German Modern Greek (1453-) 2500 14008 12753 2869 16440 15179 64421019 897052951 940023419
Expand All @@ -68,6 +71,7 @@ deu-msa German Malay (macrolanguage) 564 3723 3339 2 41 34 33449005 407965713 39
deu-nld German Dutch 11046 66228 68690 26387 157718 163499 156204242 2205120712 2342344953
deu-nor German Norwegian 3651 29080 28408 1026 7880 7686 51017401 675266045 661524476
deu-pol German Polish 5000 31008 25467 6188 38516 31606 96443936 1202484519 1111298189
deu-por German Portuguese 10000 67116 68443 7821 56554 57501 148264788 2123070857 2310849342
deu-ron German Romanian 1141 6409 6093 1112 6236 5974 60607361 791497338 857110570
deu-rus German Russian 18028 116538 99247 100323 629710 536786 73410989 928240381 850364874
deu-slv German Slovenian 659 3005 2447 7 35 26 41570897 572500671 531002712
Expand Down Expand Up @@ -120,6 +124,7 @@ eng-mya English Burmese 220 1448 1204 10751538 118720958 63785453
eng-nld English Dutch 17138 108400 111026 62387 367039 378888 325047981 4567826225 4582607424
eng-nor English Norwegian 5000 34479 33820 6264 41940 41230 135146726 1863522345 1763266796
eng-pol English Polish 11713 76000 62046 44189 281683 230039 210320051 2860689635 2510710788
eng-por English Portuguese 23215 243889 253787 204462 1296582 1297175 429437198 6632852087 6821587935
eng-ron English Romanian 6632 42043 41383 9662 62182 61687 148959401 2049428724 2106930773
eng-rus English Russian 36385 246349 208680 504175 3130009 2706675 214645928 3392370731 2954742470
eng-slv English Slovenian 2495 14074 11554 1796 10243 8404 69320844 950570923 834895987
Expand Down Expand Up @@ -264,6 +269,7 @@ pol-zho Polish Chinese 1004 5429 1012 2 11 2 20004851 152753977 36847704
por-por Portuguese Portuguese 2500 16799 16318 4783 31951 30979 44495182 324940189 324890888
por-ron Portuguese Romanian 757 4298 4210 31 161 150 90988564 1025290080 1039014561
por-rus Portuguese Russian 10797 69927 58266 10050 65095 54016 69138805 843205213 726642810
por-spa Portuguese Spanish 11875 83452 83335 56853 373767 373214 338247062 5045068645 5265684325
por-swe Portuguese Swedish 323 1692 1612 13 77 68 90547760 1285175583 1140482523
por-tur Portuguese Turkish 1794 9860 7254 76 500 365 79208095 677244614 529742537
por-ukr Portuguese Ukrainian 3372 17495 14867 1069 5671 4802 11436463 166848064 138662558
Expand Down
17 changes: 11 additions & 6 deletions data/release/v2023-09-26/released-bitexts-insufficient.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ afr-nor Afrikaans Norwegian 14 114 105 2874364 45687856 43492023
afr-pol Afrikaans Polish 22 125 111 1615233 27245093 21570093
afr-por Afrikaans Portuguese 94 494 458 1795565 29995875 29395984
afr-ron Afrikaans Romanian 2 7 8 1482972 25593970 25787300
afr-swa Afrikaans Swahili (macrolanguage) 2 31 20 3088033 42678272 39661844
afr-swe Afrikaans Swedish 5 30 28 3114165 47990831 44501557
afr-tsn Afrikaans Tswana 13 90 92 1143756 16183495 21406115
afr-tur Afrikaans Turkish 59 358 242 3727739 42566845 32789513
Expand Down Expand Up @@ -78,7 +79,7 @@ ang-deu Old English (ca. 450-1100) German 74 506 531 739 1567 1803
ang-eng Old English (ca. 450-1100) English 199 1633 1731 4 26 24 1418 2776 3272
ang-enm Old English (ca. 450-1100) Middle English (1100-1500) 28 89 100 1 3 3
ang-epo Old English (ca. 450-1100) Esperanto 59 335 333 1 3 3 632 1547 1725
ang-fao Old English (ca. 450-1100) Faroese 1 4 3
ang-fao Old English (ca. 450-1100) Faroese 1 4 3 1 0 0
ang-fin Old English (ca. 450-1100) Finnish 60 165 157 8 18 14 724 1543 1547
ang-fra Old English (ca. 450-1100) French 42 288 316 712 1491 2017
ang-gos Old English (ca. 450-1100) Gronings 1 4 3
Expand All @@ -92,7 +93,7 @@ ang-ita Old English (ca. 450-1100) Italian 10 57 54 680 1451 1745
ang-jbo Old English (ca. 450-1100) Lojban 6 18 23
ang-kur Old English (ca. 450-1100) Kurdish 4 15 14 537 978 1105
ang-lad Old English (ca. 450-1100) Ladino 19 65 78
ang-lat Old English (ca. 450-1100) Latin 27 106 83 1 3 2
ang-lat Old English (ca. 450-1100) Latin 27 106 83 1 3 2 1 0 0
ang-lfn Old English (ca. 450-1100) Lingua Franca Nova 6 22 28
ang-lij Old English (ca. 450-1100) Ligurian 1 2 4
ang-ltz Old English (ca. 450-1100) Luxembourgish 1 3 3
Expand Down Expand Up @@ -427,9 +428,9 @@ ber-ita Berber languages Italian 181 789 910 5 11 25
ber-jpn Berber languages Japanese 29 131 31 5 10 13
ber-kab Berber languages Kabyle 139 784 770
ber-kat Berber languages Georgian 1 2 4 1 3 6
ber-kur Berber languages Kurdish 3 8 13
ber-kur Berber languages Kurdish 3 8 13 1 0 0
ber-lad Berber languages Ladino 2 10 11
ber-lat Berber languages Latin 4 16 15
ber-lat Berber languages Latin 4 16 15 1 0 0
ber-lfn Berber languages Lingua Franca Nova 4 12 13
ber-lit Berber languages Lithuanian 1 5 3 6 12 35
ber-mkd Berber languages Macedonian 1 5 5 4 9 20
Expand Down Expand Up @@ -647,7 +648,7 @@ ces-fas Czech Persian 2 4 5 9415035 80281380 102082697
ces-fin Czech Finnish 88 298 293 1 3 4 50360339 600430383 487171745
ces-fry Czech Western Frisian 5 37 52 90423 356103 397944
ces-gos Czech Gronings 2 11 14
ces-grc Czech Ancient Greek (to 1453) 22 67 65
ces-grc Czech Ancient Greek (to 1453) 22 67 65 1 0 0
ces-hbs Czech Serbo-Croatian 29 108 119 65112873 560891097 569928466
ces-heb Czech Hebrew 37 162 178 3 11 8 26238574 194010865 183266018
ces-hsb Czech Upper Sorbian 170 573 590 38013 260132 261570
Expand Down Expand Up @@ -869,7 +870,7 @@ dan-tzl Danish Talossan 13 55 57
dan-ukr Danish Ukrainian 30 143 122 1 4 3 6801180 95532880 84064796
dan-urd Danish Urdu 3 10 12 628184 11106365 14144388
dan-vep Danish Veps 1 25 21
dan-vol Danish Volapük 13 45 44 2 11 10
dan-vol Danish Volapük 13 45 44 2 11 10 1 0 0
dan-xmf Danish Mingrelian 1 5 3
deu-div German Dhivehi 1 6 5 24827 603408 548485
deu-dng German Dungan 2 9 6
Expand Down Expand Up @@ -1087,6 +1088,7 @@ ell-mkd Modern Greek (1453-) Macedonian 1 3 6 3367096 31359167 30562780
ell-mlt Modern Greek (1453-) Maltese 60 148 127 4777813 102718178 84160531
ell-msa Modern Greek (1453-) Malay (macrolanguage) 9 18 23 19247897 184503055 168849488
ell-nds Modern Greek (1453-) Low German 28 88 119 90360 560153 533181
ell-nor Modern Greek (1453-) Norwegian 14 55 55 21533118 277175339 260836253
ell-pap Modern Greek (1453-) Papiamento 5 7 17 282576 4798290 5571838
ell-pol Modern Greek (1453-) Polish 121 509 470 51613014 614478092 526733844
ell-ron Modern Greek (1453-) Romanian 2 11 11 46264336 518046247 529145266
Expand Down Expand Up @@ -2054,6 +2056,7 @@ hbs-ron Serbo-Croatian Romanian 1 7 7 68549042 586505598 678253781
hbs-rue Serbo-Croatian Rusyn 1 42 36
hbs-slv Serbo-Croatian Slovenian 39 130 111 45209193 446989984 435127760
hbs-sme Serbo-Croatian Northern Sami 4 19 16 24240 106302 89984
hbs-swe Serbo-Croatian Swedish 2 5 7 37131475 367479337 379539645
hbs-tat Serbo-Croatian Tatar 2 25 25 271021 3748551 3073533
hbs-tok Serbo-Croatian Toki Pona 3 9 24
hbs-urd Serbo-Croatian Urdu 1 2 3 1334624 19111410 25967830
Expand Down Expand Up @@ -2086,6 +2089,7 @@ heb-ron Hebrew Romanian 8 35 56 26577504 185777184 228634691
heb-sjn Hebrew Sindarin 10 27 29 3 8 8
heb-slv Hebrew Slovenian 4 15 16 15004636 109948372 111620424
heb-stq Hebrew Saterfriesisch 2 4 6
heb-swe Hebrew Swedish 6 22 28 20435696 148417014 166917221
heb-swg Hebrew Swabian 1 2 3
heb-syc Hebrew Classical Syriac 1 5 5
heb-tgl Hebrew Tagalog 15 64 76 1410815 16832485 21685474
Expand Down Expand Up @@ -2362,6 +2366,7 @@ ita-slv Italian Slovenian 79 377 332 31782491 477512726 408315845
ita-sqi Italian Albanian 56 304 316 18 117 115 8619608 110171133 116437811
ita-swa Italian Swahili (macrolanguage) 22 99 66 1 3 2 2767434 41546024 38207835
ita-tat Italian Tatar 52 347 366 256744 4029605 2977928
ita-tgl Italian Tagalog 93 477 498 2182798 31541584 34950122
ita-tha Italian Thai 15 68 22 6487036 91646302 29356807
ita-tlh Italian Klingon 140 620 416 5 20 15 127 2395 1341
ita-tmr Italian Jewish Babylonian Aramaic (ca. 200-1200 CE) 5 18 13
Expand Down
5 changes: 5 additions & 0 deletions data/release/v2023-09-26/released-bitexts-min1000.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,11 @@ dan-eng Danish English 12678 79295 79827 20089 123471 124078 164950651 236653823
dan-epo Danish Esperanto 13144 79397 75422 19236 117228 111170 1573951 15401229 15224020
dan-fin Danish Finnish 2665 13452 11009 1854 10314 8348 49411725 778227129 591479216
dan-fra Danish French 1731 9307 10245 89 601 690 77220719 1240133470 1433684311
dan-nld Danish Dutch 1643 8485 8952 67 338 365 71005999 1160485402 1258693815
dan-nor Danish Norwegian 1311 8343 8216 1050 6607 6561 39503372 541325880 535545627
dan-rus Danish Russian 1713 9788 8376 1065 5999 5207 19444845 180706655 164465722
dan-spa Danish Spanish 5000 30181 29832 5165 31534 31087 81687841 1280222017 1487592915
dan-swe Danish Swedish 2348 12542 12292 1440 7859 7730 67424560 1051984859 1026298995
deu-deu German German 2500 17212 17358 2127 14410 14645 402977 4134396 4107924
deu-ell German Modern Greek (1453-) 2500 14008 12753 2869 16440 15179 64421019 897052951 940023419
deu-eng German English 32565 247086 250438 289779 2004590 2005962 775736768 11191697189 11879775260
Expand All @@ -72,6 +74,7 @@ deu-nds German Low German 9999 62145 63862 7972 50286 51767 146720 1364034 14319
deu-nld German Dutch 11046 66228 68690 26387 157718 163499 156204242 2205120712 2342344953
deu-nor German Norwegian 3651 29080 28408 1026 7880 7686 51017401 675266045 661524476
deu-pol German Polish 5000 31008 25467 6188 38516 31606 96443936 1202484519 1111298189
deu-por German Portuguese 10000 67116 68443 7821 56554 57501 148264788 2123070857 2310849342
deu-ron German Romanian 1141 6409 6093 1112 6236 5974 60607361 791497338 857110570
deu-run German Rundi 1752 8475 5109 1192 6287 3869 345565 5691274 5176341
deu-rus German Russian 18028 116538 99247 100323 629710 536786 73410989 928240381 850364874
Expand Down Expand Up @@ -131,6 +134,7 @@ eng-nld English Dutch 17138 108400 111026 62387 367039 378888 325047981 45678262
eng-nor English Norwegian 5000 34479 33820 6264 41940 41230 135146726 1863522345 1763266796
eng-pam English Pampanga 1000 5621 4989 494 2851 2504 30 78 64
eng-pol English Polish 11713 76000 62046 44189 281683 230039 210320051 2860689635 2510710788
eng-por English Portuguese 23215 243889 253787 204462 1296582 1297175 429437198 6632852087 6821587935
eng-ron English Romanian 6632 42043 41383 9662 62182 61687 148959401 2049428724 2106930773
eng-run English Rundi 1703 8221 4930 3 29 17 1024349 12413204 11015958
eng-rus English Russian 36385 246349 208680 504175 3130009 2706675 214645928 3392370731 2954742470
Expand Down Expand Up @@ -298,6 +302,7 @@ pol-ukr Polish Ukrainian 2706 11176 11454 6898 27996 28750 11773783 139823626 13
pol-zho Polish Chinese 1004 5429 1012 2 11 2 20004851 152753977 36847704
por-por Portuguese Portuguese 2500 16799 16318 4783 31951 30979 44495182 324940189 324890888
por-rus Portuguese Russian 10797 69927 58266 10050 65095 54016 69138805 843205213 726642810
por-spa Portuguese Spanish 11875 83452 83335 56853 373767 373214 338247062 5045068645 5265684325
por-tgl Portuguese Tagalog 1777 10677 11473 3 17 16 2296650 32457112 35828948
por-tok Portuguese Toki Pona 1719 11414 16030 1023 8613 12958
por-tur Portuguese Turkish 1794 9860 7254 76 500 365 79208095 677244614 529742537
Expand Down
6 changes: 6 additions & 0 deletions data/release/v2023-09-26/released-bitexts-min200.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ bul-bul Bulgarian Bulgarian 219 1078 1101 500820 3423840 3434443
bul-deu Bulgarian German 317 1639 1827 47383235 617102672 615370548
bul-eng Bulgarian English 10000 56740 60705 7835 44204 47335 113270896 1453244814 1528332996
bul-epo Bulgarian Esperanto 588 3325 3417 1 8 6 1436274 15159296 15206768
bul-fra Bulgarian French 451 2724 3243 43704625 475998935 536869706
bul-ita Bulgarian Italian 2500 13344 14074 4006 21635 22681 43149114 548361970 584782283
bul-jpn Bulgarian Japanese 322 1955 325 2 10 2 8599663 84257976 15833135
bul-rus Bulgarian Russian 1247 6556 6155 1001 5184 4806 44461448 458102211 419678955
Expand Down Expand Up @@ -105,10 +106,12 @@ dan-fra Danish French 1731 9307 10245 89 601 690 77220719 1240133470 1433684311
dan-hun Danish Hungarian 882 4967 3888 6 26 18 39096324 568567353 515841359
dan-ita Danish Italian 286 1876 1885 4 16 18 63225160 1077789214 1171882384
dan-jpn Danish Japanese 927 5000 932 10 62 10 8467609 97226762 15999268
dan-nld Danish Dutch 1643 8485 8952 67 338 365 71005999 1160485402 1258693815
dan-nor Danish Norwegian 1311 8343 8216 1050 6607 6561 39503372 541325880 535545627
dan-por Danish Portuguese 876 4281 4368 4 33 24 73045085 1099601319 1219803678
dan-rus Danish Russian 1713 9788 8376 1065 5999 5207 19444845 180706655 164465722
dan-spa Danish Spanish 5000 30181 29832 5165 31534 31087 81687841 1280222017 1487592915
dan-swe Danish Swedish 2348 12542 12292 1440 7859 7730 67424560 1051984859 1026298995
dan-tur Danish Turkish 770 3555 2680 4 16 12 19482641 182994406 150078891
dan-yid Danish Yiddish 200 968 1053 11 58 60 79581 399299 635391
dan-zho Danish Chinese 250 1400 265 12949875 111175424 23597243
Expand Down Expand Up @@ -150,6 +153,7 @@ deu-nds German Low German 9999 62145 63862 7972 50286 51767 146720 1364034 14319
deu-nld German Dutch 11046 66228 68690 26387 157718 163499 156204242 2205120712 2342344953
deu-nor German Norwegian 3651 29080 28408 1026 7880 7686 51017401 675266045 661524476
deu-pol German Polish 5000 31008 25467 6188 38516 31606 96443936 1202484519 1111298189
deu-por German Portuguese 10000 67116 68443 7821 56554 57501 148264788 2123070857 2310849342
deu-ron German Romanian 1141 6409 6093 1112 6236 5974 60607361 791497338 857110570
deu-run German Rundi 1752 8475 5109 1192 6287 3869 345565 5691274 5176341
deu-rus German Russian 18028 116538 99247 100323 629710 536786 73410989 928240381 850364874
Expand Down Expand Up @@ -250,6 +254,7 @@ eng-ota English Ottoman Turkish (1500-1928) 696 3704 3433 12 81 72
eng-pam English Pampanga 1000 5621 4989 494 2851 2504 30 78 64
eng-pms English Piemontese 270 1714 1910 45 299 285
eng-pol English Polish 11713 76000 62046 44189 281683 230039 210320051 2860689635 2510710788
eng-por English Portuguese 23215 243889 253787 204462 1296582 1297175 429437198 6632852087 6821587935
eng-prg English Prussian 219 1444 1277
eng-que English Quechua 258 1386 764 2853672 34438187 22640942
eng-rom English Romany 706 3961 4494 23887 447988 458579
Expand Down Expand Up @@ -593,6 +598,7 @@ pol-zho Polish Chinese 1004 5429 1012 2 11 2 20004851 152753977 36847704
por-por Portuguese Portuguese 2500 16799 16318 4783 31951 30979 44495182 324940189 324890888
por-ron Portuguese Romanian 757 4298 4210 31 161 150 90988564 1025290080 1039014561
por-rus Portuguese Russian 10797 69927 58266 10050 65095 54016 69138805 843205213 726642810
por-spa Portuguese Spanish 11875 83452 83335 56853 373767 373214 338247062 5045068645 5265684325
por-swe Portuguese Swedish 323 1692 1612 13 77 68 90547760 1285175583 1140482523
por-tgl Portuguese Tagalog 1777 10677 11473 3 17 16 2296650 32457112 35828948
por-tok Portuguese Toki Pona 1719 11414 16030 1023 8613 12958
Expand Down
Loading

0 comments on commit f691911

Please sign in to comment.