refactor basic tokenizer

sagorbrur · Jul 10, 2023 · 73519bc · 73519bc
1 parent b1c63e1
commit 73519bc
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/bnlp/tokenizer/basic.py b/bnlp/tokenizer/basic.py
@@ -65,7 +65,6 @@ class BasicTokenizer:
     def tokenize(self, text):
         """Tokenizes a piece of text."""
         text = convert_to_unicode(text)
-        # handle (.) in bangla text
 
         orig_tokens = whitespace_tokenize(text)
         # print("original tokens: ", orig_tokens)
@@ -75,7 +74,6 @@ def tokenize(self, text):
 
         # print("split tokens: ", split_tokens)
         output_tokens = whitespace_tokenize(" ".join(split_tokens))
-        # get (.) back in output tokens
         return output_tokens
 
     def _run_strip_accents(self, text):