fix(itn): 七八公斤 => 7~8kg, 七八百块 => 700~800块 (#116)

wenet-e2e · Oct 12, 2023 · fc1b2ca · fc1b2ca
1 parent 75ac443
commit fc1b2ca
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 0 deletions.
diff --git a/itn/chinese/data/measure/units_en.tsv b/itn/chinese/data/measure/units_en.tsv
@@ -24,6 +24,7 @@
 千卡	kcal
 千克力	kgf
 千克	kg
+公斤	kg
 千赫兹	khz
 平方千米	km²
 公里	km

diff --git a/itn/chinese/rules/cardinal.py b/itn/chinese/rules/cardinal.py
@@ -72,6 +72,13 @@ def build_tagger(self):
                   (number + accep('亿') + delete('零').ques).ques + number)
         # 负的xxx 1.11, 1.01
         number = sign.ques + number + (dot + digits.plus).ques
+        # 五六万，三五千，六七百，三四十
+        number |= add_weight(
+            (digit + insert("0~") + digit + cross("十", "0")) |
+            (digit + insert("00~") + digit + cross("百", "00")) |
+            (digit + insert("000~") + digit + cross("千", "000")) |
+            (digit + insert("0000~") + digit + cross("万", "0000")), -1.0
+        )
         self.number = number.optimize()
 
         # 十/百/千/万
@@ -87,6 +94,13 @@ def build_tagger(self):
             (number_exclude_0_to_9 | digits) +
             (dot + digits.plus).plus
         )
+        # 五六万，三五千，六七百，三四十
+        number_exclude_0_to_9 |= add_weight(
+            (digit + insert("0~") + digit + cross("十", "0")) |
+            (digit + insert("00~") + digit + cross("百", "00")) |
+            (digit + insert("000~") + digit + cross("千", "000")) |
+            (digit + insert("0000~") + digit + cross("万", "0000")), -1.0
+        )
         self.number_exclude_0_to_9 = (sign.ques + number_exclude_0_to_9).optimize()  # noqa
 
         # cardinal string like 127.0.0.1, used in ID, IP, etc.

diff --git a/itn/chinese/rules/measure.py b/itn/chinese/rules/measure.py
@@ -31,6 +31,7 @@ def __init__(self, exclude_one=True, enable_0_to_9=True):
     def build_tagger(self):
         units_en = string_file('itn/chinese/data/measure/units_en.tsv')
         units_zh = string_file('itn/chinese/data/measure/units_zh.tsv')
+        digit = string_file('itn/chinese/data/number/digit.tsv')  # 1 ~ 9
         sign = string_file('itn/chinese/data/number/sign.tsv')    # + -
         to = cross('到', '~') | cross('到百分之', '~')
 
@@ -48,6 +49,8 @@ def build_tagger(self):
 
         # 十千米每小时 => 10km/h, 十一到一百千米每小时 => 11~100km/h
         measure = number + (to + number).ques + units
+        # 七八块钱
+        measure |= add_weight(digit + insert("~") + digit + units, -1.0)
         tagger = insert('value: "') + (measure | percent) + insert('"')
 
         # 每小时十千米 => 10km/h, 每小时三十到三百一十一千米 => 30~311km/h

diff --git a/itn/chinese/test/data/cardinal.txt b/itn/chinese/test/data/cardinal.txt
@@ -1,3 +1,8 @@
 幺幺零 => 110
 幺二七点零点零点幺 => 127.0.0.1
 这是手机一八五四四一三九一二一 => 这是手机18544139121
+三五百 => 300~500
+三五千 => 3000~5000
+三五万 => 30000~50000
+三四万 => 30000~40000
+五六十 => 50~60
diff --git a/itn/chinese/test/data/measure.txt b/itn/chinese/test/data/measure.txt
@@ -32,3 +32,8 @@
 百分之三十一到百分之百 => 31~100%
 十一到一百千米每小时 => 11~100km/h
 每小时三十到三百一十一千米 => 30~311km/h
+七八公斤 => 7~8kg
+五六十块钱 => 50~60块钱
+三五百公里 => 300~500km
+八九千美元 => $8000~9000
+三四万吨 => 30000~40000吨