From 4533ed014fc4a22d3f7123c8f2ab3425ff7afb2f Mon Sep 17 00:00:00 2001
From: duyalei <>
Date: Mon, 23 Sep 2024 16:34:31 +0800
Subject: [PATCH] add full-width punctuations as end of the sentence

---
 src/pipecat/utils/string.py |  3 ++-
 tests/test_ai_services.py   | 13 +++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py
index a47db6c5c..cf9a22ad8 100644
--- a/src/pipecat/utils/string.py
+++ b/src/pipecat/utils/string.py
@@ -14,7 +14,8 @@
     (?<!Mr|Ms|Dr)    # Negative lookbehind: not preceded by Mr, Ms, Dr (combined bc. length is the same)
     (?<!Mrs)         # Negative lookbehind: not preceded by "Mrs"
     (?<!Prof)        # Negative lookbehind: not preceded by "Prof"
-    [\.\?\!:]        # Match a period, question mark, exclamation point, or colon
+    [\.\?\!:;]|      # Match a period, question mark, exclamation point, colon, or semicolon
+    [。？！：；]       # the full-width version (mainly used in East Asian languages such as Chinese)
     $                # End of string
 """
 ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
diff --git a/tests/test_ai_services.py b/tests/test_ai_services.py
index fb00fc893..f539a44ce 100644
--- a/tests/test_ai_services.py
+++ b/tests/test_ai_services.py
@@ -32,6 +32,7 @@ async def test_endofsentence(self):
         assert match_endofsentence("This is a sentence! ")
         assert match_endofsentence("This is a sentence?")
         assert match_endofsentence("This is a sentence:")
+        assert match_endofsentence("This is a sentence;")
         assert not match_endofsentence("This is not a sentence")
         assert not match_endofsentence("This is not a sentence,")
         assert not match_endofsentence("This is not a sentence, ")
@@ -43,6 +44,18 @@ async def test_endofsentence(self):
         assert not match_endofsentence("America, or the U.")  # U.S.A.
         assert not match_endofsentence("It still early, it's 3:00 a.")  # 3:00 a.m.
 
+    async def test_endofsentence_zh(self):
+        chinese_sentences = [
+            "你好。",
+            "你好！",
+            "吃了吗？",
+            "安全第一；",
+            "他说：",
+        ]
+        for i in chinese_sentences:
+            assert match_endofsentence(i)
+        assert not match_endofsentence("你好，")
+
 
 if __name__ == "__main__":
     unittest.main()