Fix fullwidth to remove ; for fix invalid convert with HTML Entity …

…case.
longbridge · Dec 25, 2020 · d528a09 · d528a09
1 parent f5c5e96
commit d528a09
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/fullwidth.go b/fullwidth.go
@@ -18,7 +18,7 @@ var (
 	}
 
 	spcialPunctuations = `[.:]`
-	normalPunctuations = `[,;\!\?~]`
+	normalPunctuations = `[,\!\?~]`
 
 	punctuationWithLeftCJKRe        = regexp.MustCompile(normalPunctuations + `[` + cjk + `]+`)
 	punctuationWithRightCJKRe       = regexp.MustCompile(`[` + cjk + `]+` + normalPunctuations)

diff --git a/fullwidth_test.go b/fullwidth_test.go
@@ -13,6 +13,8 @@ func Test_fullwidth(t *testing.T) {
 		"蚂蚁疾奔:蚂蚁集团两地上市~全速推进!":                                                        "蚂蚁疾奔：蚂蚁集团两地上市～全速推进！",
 		"蚂蚁集团是阿里巴巴(BABA.N)旗下金融科技子公司":                                                 "蚂蚁集团是阿里巴巴 (BABA.N) 旗下金融科技子公司",
 		"Dollar的演示 $阿里巴巴.US$ 股票标签":                                                   "Dollar 的演示 $阿里巴巴.US$ 股票标签",
+		// https://developer.mozilla.org/en-US/docs/Glossary/Entity
+		"确保&quot;&gt;HTML Entity&lt;&quot;的字符&#34;不会被处理&#34; Ruby&amp;Go": "确保&quot;&gt;HTML Entity&lt;&quot;的字符&#34;不会被处理&#34; Ruby&amp;Go",
 	}
 
 	for source, exptected := range cases {