Dev (#2)

* update test and travis

* fixed encoding error when opening file with python2

.travis.yml

@@ -1,5 +1,6 @@
 language: python
 python:
+  - "2.7"
   - "3.5"
   - "3.6"
 install:

bert_slot_tokenizer/__init__.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # Created by Ross on 2019/7/31
 import json
-
+import io
 from bert_slot_tokenizer.bert_tokenizer import tokenization
 
 
@@ -31,7 +31,7 @@ def __init__(self, vacab_file, do_lower_case=True):
 
     @staticmethod
     def parse_json(file):
-        with open(file, 'r', encoding='utf-8') as f:
+        with io.open(file, 'r', encoding='utf-8') as f:
             data = json.load(f)
 
         texts = []

bert_slot_tokenizer/bert_tokenizer/tokenization.py

@@ -21,7 +21,7 @@
 import collections
 import re
 import unicodedata
-
+import io
 import six
 
 
@@ -134,7 +134,7 @@ def load_vocab(vocab_file):
     index = 0
 
     # with tf.gfile.GFile(vocab_file, "r") as reader:  # modified here
-    with open(vocab_file, "r") as reader:
+    with io.open(vocab_file, "r", encoding='utf-8') as reader:
         while True:
             token = convert_to_unicode(reader.readline())
             if not token:

bert_slot_tokenizer/main.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 
 def main():

setup.py

@@ -27,7 +27,7 @@
 def get_version():
     main_file = os.path.join(CURDIR, "bert_slot_tokenizer", "main.py")
     _version_re = re.compile(r"__version__\s+=\s+(?P<version>.*)")
-    with open(main_file, "r", encoding="utf8") as f:
+    with io.open(main_file, "r", encoding="utf8") as f:
         match = _version_re.search(f.read())
         version = match.group("version") if match is not None else '"unknown"'
     return str(ast.literal_eval(version))

tests/test_project.py

@@ -44,7 +44,7 @@ def test_import(self):
 
     def test_project(self):
 
-        sc = SlotConverter('test_data/example_vocab.txt', do_lower_case=True)
+        sc = SlotConverter('tests/test_data/example_vocab.txt', do_lower_case=True)
 
         if six.PY3:
             token1, iob_slot1 = sc.convert2iob(UnitTests.test_case1['text'], UnitTests.test_case1['slots'])