From c7513f35e2b8cff13e398a625471b1297b9da627 Mon Sep 17 00:00:00 2001 From: HRUSHIKESH DOKALA <96101829+Hk669@users.noreply.github.com> Date: Wed, 5 Jun 2024 20:44:20 +0530 Subject: [PATCH] deprecated the __version__ check when loading (#8) --- bpetokenizer/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bpetokenizer/base.py b/bpetokenizer/base.py index f6f7cd8..e0d8123 100644 --- a/bpetokenizer/base.py +++ b/bpetokenizer/base.py @@ -138,7 +138,6 @@ def load(self, file_name, mode="json"): special_tokens = {} idx = 256 with open(file_name, 'r', encoding="utf-8") as f: - assert f.readline().strip() == __version__ self.pattern = f.readline().strip().split() num_special = int(f.readline().strip()) # no of lines of special_tokens for _ in range(num_special): @@ -159,7 +158,6 @@ def load(self, file_name, mode="json"): import json with open(file_name, "r", encoding="utf-8") as f: data = json.load(f) - assert data["version"] == __version__ pattern = data["pattern"] pattern_regex = re.compile(r'regex.Regex\("(.+)", flags=(regex\.\w+)\)') match = pattern_regex.match(pattern)