better handling of Latvian

adbar · Nov 5, 2024 · fac72b5 · fac72b5
1 parent 97c3285
commit fac72b5
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 5 deletions.
diff --git a/simplemma/strategies/defaultrules/lv.py b/simplemma/strategies/defaultrules/lv.py
@@ -31,15 +31,15 @@
     # fallback
     re.compile(r"(?:as|ai|ā|ām|ās)$"): "a",
     re.compile(r"(?:ei|es|ē|ēm|ēs)$"): "e",
+    re.compile(r"(?:is|im|ī|iem|īs)$"): "is",
     # re.compile(r"(?:os|us)$"): "s",
-    # re.compile(r"(?:is|im|ī|iem|īs)$"): "is",
     # re.compile(r"(?:ēto|ēts)$"): "ēt",
 }
 
 
 def apply_lv(token: str) -> Optional[str]:
     "Apply pre-defined rules for Latvian."
-    if len(token) < 5 or token[0].isupper():
+    if len(token) < 4:
         return None
 
     return apply_rules(token, DEFAULT_RULES)
diff --git a/simplemma/strategies/dictionaries/data/lv.plzma b/simplemma/strategies/dictionaries/data/lv.plzma
diff --git a/simplemma/strategies/greedy_dictionary_lookup.py b/simplemma/strategies/greedy_dictionary_lookup.py
@@ -7,7 +7,7 @@
 from .dictionaries.dictionary_factory import DefaultDictionaryFactory, DictionaryFactory
 from .lemmatization_strategy import LemmatizationStrategy
 
-SHORTER_GREEDY = {"bg", "et", "fi"}
+SHORTER_GREEDY = {"bg", "et", "fi", "lv"}
 
 
 class GreedyDictionaryLookupStrategy(LemmatizationStrategy):

diff --git a/training/dictionary_pickler.py b/training/dictionary_pickler.py
@@ -19,7 +19,7 @@
 
 LOGGER = logging.getLogger(__name__)
 
-INPUT_PUNCT = re.compile(r"[,:*/\+_]|^-|-\t")
+INPUT_PUNCT = re.compile(r"[,:*/\+_]|.+-$|.+-\t")
 SAFE_LIMIT = {
     "cs",
     "da",
@@ -31,6 +31,7 @@
     "ga",
     "hu",
     "it",
+    "lv",
     "pl",
     "pt",
     "ru",
@@ -86,7 +87,7 @@ def _read_dict(
                 and columns[1] != columns[0]
             ):
                 rule = DEFAULT_RULES[langcode](columns[1])
-                if rule is not None and rule != columns[1]:
+                if rule and rule != columns[0]:
                     print(columns[1], columns[0], rule)
             # process
             if columns[1] in mydict and mydict[columns[1]] != columns[0]: