Add whitespace cleaner.

mltb2/text.py

@@ -4,7 +4,8 @@
 
 """Text specific module."""
 
-from typing import Dict, Final, Tuple
+import re
+from typing import Dict, Final, Pattern, Tuple
 
 INVISIBLE_CHARACTERS: Final[Tuple[str, ...]] = (
     "\u200b",  # Zero Width Space (ZWSP) https://www.compart.com/en/unicode/U+200b
@@ -36,11 +37,15 @@
 
 SPECIAL_WHITESPACES_TRANS: Final[Dict[int, str]] = str.maketrans({char: " " for char in SPECIAL_WHITESPACES})
 
+INVISIBLE_CHARACTERS_AND_SPECIAL_WHITESPACES_TRANS = {**SPECIAL_WHITESPACES_TRANS, **INVISIBLE_CHARACTERS_TRANS}
+
+MULTI_SPACE_PATTERN: Pattern = re.compile(r" {2,}")
+
 
 def remove_invisible_characters(text: str) -> str:
     """Remove invisible characters from text.
 
-    The invisible characters are defined in the constant `INVISIBLE_CHARACTERS`.
+    The invisible characters are defined in the constant ``INVISIBLE_CHARACTERS``.
 
     Args:
         text: The text from which the invisible characters are to be removed.
@@ -54,7 +59,7 @@ def remove_invisible_characters(text: str) -> str:
 def has_invisible_characters(text: str) -> bool:
     """Check if text contains invisible characters.
 
-    The invisible characters are defined in the constant `INVISIBLE_CHARACTERS`.
+    The invisible characters are defined in the constant ``INVISIBLE_CHARACTERS``.
 
     Args:
         text: The text to check.
@@ -68,7 +73,7 @@ def has_invisible_characters(text: str) -> bool:
 def replace_special_whitespaces(text: str) -> str:
     """Replace special whitespaces with normal whitespaces.
 
-    The special whitespaces are defined in the constant `SPECIAL_WHITESPACES`.
+    The special whitespaces are defined in the constant ``SPECIAL_WHITESPACES``.
 
     Args:
         text: The text from which the special whitespaces are to be replaced.
@@ -82,7 +87,7 @@ def replace_special_whitespaces(text: str) -> str:
 def has_special_whitespaces(text: str) -> bool:
     """Check if text contains special whitespaces.
 
-    The special whitespaces are defined in the constant `SPECIAL_WHITESPACES`.
+    The special whitespaces are defined in the constant ``SPECIAL_WHITESPACES``.
 
     Args:
         text: The text to check.
@@ -91,3 +96,38 @@ def has_special_whitespaces(text: str) -> bool:
         ``True`` if the text contains special whitespaces, ``False`` otherwise.
     """
     return any(char in text for char in SPECIAL_WHITESPACES)
+
+
+def replace_multiple_whitespaces(text: str) -> str:
+    """Replace multiple whitespaces with single whitespace.
+
+    Args:
+        text: The text from which the multiple whitespaces are to be replaced.
+
+    Returns:
+        The cleaned text.
+    """
+    return MULTI_SPACE_PATTERN.sub(" ", text)
+
+
+def clean_all_invisible_chars_and_whitespaces(text: str) -> str:
+    """Clean text form invisible characters and whitespaces.
+
+    - Remove invisible characters from text.
+    - Replace special whitespaces with normal whitespaces.
+    - Replace multiple whitespaces with single whitespace.
+    - Remove leading and trailing whitespaces.
+
+    The invisible characters are defined in the constant ``INVISIBLE_CHARACTERS``.
+    The special whitespaces are defined in the constant ``SPECIAL_WHITESPACES``.
+
+    Args:
+        text: The text to clean.
+
+    Rteturns:
+        The cleaned text.
+    """
+    text = text.translate(INVISIBLE_CHARACTERS_AND_SPECIAL_WHITESPACES_TRANS)
+    text = replace_multiple_whitespaces(text)
+    text = text.strip()
+    return text

tests/test_text.py

@@ -7,9 +7,11 @@
 from mltb2.text import (
     INVISIBLE_CHARACTERS,
     SPECIAL_WHITESPACES,
+    clean_all_invisible_chars_and_whitespaces,
     has_invisible_characters,
     has_special_whitespaces,
     remove_invisible_characters,
+    replace_multiple_whitespaces,
     replace_special_whitespaces,
 )
 
@@ -74,3 +76,39 @@ def test_has_special_whitespaces_false():
     text = "Hello you!"
     result = has_special_whitespaces(text)
     assert not result
+
+
+def test_replace_multiple_whitespaces():
+    text = "Hello  World  !"
+    result = replace_multiple_whitespaces(text)
+    assert result == "Hello World !"
+
+
+def test_replace_multiple_whitespaces_empty():
+    text = ""
+    result = replace_multiple_whitespaces(text)
+    assert result == ""
+
+
+def test_replace_multiple_whitespaces_empty_result():
+    text = "  "
+    result = replace_multiple_whitespaces(text)
+    assert result == " "
+
+
+def test_replace_multiple_whitespaces_one_space():
+    text = " "
+    result = replace_multiple_whitespaces(text)
+    assert result == " "
+
+
+def test_clean_all_invisible_chars_and_whitespaces():
+    text = " Hello\u200bWorld\u00ad! How\u2007  are you? "
+    result = clean_all_invisible_chars_and_whitespaces(text)
+    assert result == "HelloWorld! How are you?"
+
+
+def test_clean_all_invisible_chars_and_whitespaces_empty_result():
+    text = " \u200b\u00ad\u2007   "
+    result = clean_all_invisible_chars_and_whitespaces(text)
+    assert result == ""