Drop unnecessary ngram files for Chinese, Japanese and Korean (pemist…

…ahl#101)
fvasco · Apr 28, 2022 · 4a13f7d · 4a13f7d
1 parent a18acfc
commit 4a13f7d
Show file tree

Hide file tree

Showing 13 changed files with 4 additions and 16 deletions.
diff --git a/src/main/kotlin/com/github/pemistahl/lingua/api/LanguageDetector.kt b/src/main/kotlin/com/github/pemistahl/lingua/api/LanguageDetector.kt
@@ -421,11 +421,11 @@ class LanguageDetector internal constructor(
             synchronized(modelCache) {
                 modelCache[language]?.let { return it }
             }
-            val jsonLanguageModels: Sequence<JsonLanguageModel> = (1..5).asSequence().map { ngramLength ->
-                val fileName = "${Ngram.getNgramNameByLength(ngramLength)}s.json"
-                val filePath = "/language-models/${language.isoCode639_1}/$fileName"
+            val jsonLanguageModels: Sequence<JsonLanguageModel> = (1..5).asSequence().mapNotNull { ngramLength ->
+                val filePath =
+                    "/language-models/${language.isoCode639_1}/${Ngram.getNgramNameByLength(ngramLength)}s.json"
                 Language::class.java.getResourceAsStream(filePath)
-                    .use { Json.decodeFromStream(JsonLanguageModel.serializer(), it) }
+                    ?.use { Json.decodeFromStream(JsonLanguageModel.serializer(), it) }
             }
             val model = TrainingDataLanguageModel.fromJson(language, jsonLanguageModels)
             synchronized(modelCache) {

diff --git a/src/main/resources/language-models/ja/bigrams.json b/src/main/resources/language-models/ja/bigrams.json
diff --git a/src/main/resources/language-models/ja/fivegrams.json b/src/main/resources/language-models/ja/fivegrams.json
diff --git a/src/main/resources/language-models/ja/quadrigrams.json b/src/main/resources/language-models/ja/quadrigrams.json
diff --git a/src/main/resources/language-models/ja/trigrams.json b/src/main/resources/language-models/ja/trigrams.json
diff --git a/src/main/resources/language-models/ko/bigrams.json b/src/main/resources/language-models/ko/bigrams.json
diff --git a/src/main/resources/language-models/ko/fivegrams.json b/src/main/resources/language-models/ko/fivegrams.json
diff --git a/src/main/resources/language-models/ko/quadrigrams.json b/src/main/resources/language-models/ko/quadrigrams.json
diff --git a/src/main/resources/language-models/ko/trigrams.json b/src/main/resources/language-models/ko/trigrams.json
diff --git a/src/main/resources/language-models/zh/bigrams.json b/src/main/resources/language-models/zh/bigrams.json
diff --git a/src/main/resources/language-models/zh/fivegrams.json b/src/main/resources/language-models/zh/fivegrams.json
diff --git a/src/main/resources/language-models/zh/quadrigrams.json b/src/main/resources/language-models/zh/quadrigrams.json
diff --git a/src/main/resources/language-models/zh/trigrams.json b/src/main/resources/language-models/zh/trigrams.json