-
Notifications
You must be signed in to change notification settings - Fork 189
/
test_remove_specific_chars_mapper.py
49 lines (39 loc) · 1.66 KB
/
test_remove_specific_chars_mapper.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import unittest
from data_juicer.core.data import NestedDataset as Dataset
from data_juicer.ops.mapper.remove_specific_chars_mapper import \
RemoveSpecificCharsMapper
from data_juicer.utils.unittest_utils import DataJuicerTestCaseBase
class RemoveSpecificCharsMapperTest(DataJuicerTestCaseBase):
def setUp(self):
self.op = RemoveSpecificCharsMapper()
def _run_helper(self, samples):
dataset = Dataset.from_list(samples)
dataset = dataset.map(self.op.process, batch_size=2)
for data in dataset:
self.assertEqual(data['text'], data['target'])
def test_complete_html_text(self):
samples = [
{
'text': '这是一个干净的文本。Including Chinese and English.',
'target': '这是一个干净的文本。Including Chinese and English.',
},
{
'text': '◆●■►▼▲▴∆▻▷❖♡□',
'target': '',
},
{
'text': '►This is a dirty text ▻ 包括中文和英文',
'target': 'This is a dirty text 包括中文和英文',
},
{
'text': '多个●■►▼这样的特殊字符可以►▼▲▴∆吗?',
'target': '多个这样的特殊字符可以吗?',
},
{
'text': '未指定的●■☛₨➩►▼▲特殊字符会☻▷❖被删掉吗??',
'target': '未指定的☛₨➩特殊字符会☻被删掉吗??',
},
]
self._run_helper(samples)
if __name__ == '__main__':
unittest.main()