Merge pull request #12 from ShoukanLabs/dev

Add manual tagging for phonemes and languages

VoPho/engine.py

@@ -110,10 +110,13 @@ def phonemize_for_language(self, text, lang):
         :param lang: The language ID for phonemization
         :return: Phonemized text, or original text wrapped in <??> tags if language is not supported
         """
-        phonemizer = self.get_phonemizer(lang)
-        if phonemizer:
-            return phonemizer.phonemize(text)
-        return f"<??>{text}</??>"  # Return original text if no phonemizer available
+        if lang != "phoneme":
+            phonemizer = self.get_phonemizer(lang)
+            if phonemizer:
+                return phonemizer.phonemize(text)
+            return f"<??>{text}</??>"  # Return original text if no phonemizer available
+        else:
+            return text
 
     def phonemize(self, input_text, output_dict=False):
         """

VoPho/langtokenizers/multicoded.py

@@ -33,6 +33,7 @@
     'mr': 'red',
     'he': 'white',
     'th': 'blue',
+    'phoneme': 'blue',
     '??': 'red'
 }
 
@@ -269,12 +270,28 @@ def _group_segments(self, text):
         return ''.join(grouped_segments).replace("<punctuation>", "").replace("</punctuation>", " ")
 
     def tokenize(self, text, group=True):
-        result = self._tokenize(text)
+        # Split the input text into segments based on existing tags
+        pattern = r'(<\w+>.*?</\w+>)|([^<]+)'  # Matches either tagged segments or untagged text
+        segments = re.findall(pattern, text)
+
+        processed_segments = []
+
+        for tagged_segment, untagged_segment in segments:
+            if tagged_segment:  # If this segment is already tagged, just add it
+                processed_segments.append(tagged_segment)
+            else:  # If the segment is untagged, process it as usual
+                result = self._tokenize(untagged_segment)
+                processed_segments.append(result)
+
+        result = ''.join(processed_segments)
+
         if group:
             result = self._group_segments(result)
+
         if "<??>" in result:
             warnings.warn(
                 "Your output contains tokenization errors. We were unable to detect a language or writing system, or there was an error in processing.")
+
         return result
 
 

examples/phonemize_texts.py

@@ -1,7 +1,7 @@
 from VoPho.engine import Phonemizer
 from time import time
 
-input_text = "I suppose i can, dont take my word for it though. 音素のテストを行うことは、発音の理解を深めるために重要です。"
+input_text = "<phoneme>I suppose i can</phoneme>, dont take my word for it though. 音素のテストを行うことは、発音の理解を深めるために重要です。"
 
 engine = Phonemizer()
 start = time()

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "VoPho"
-version = "0.0.9"
+version = "0.0.10"
 description = "An easy to use Multilingual phonemization meta-library"
 readme = "README.md"
 authors = [