diff --git a/tests/unit/tokenizer/test_cohere_hf_tokenizer.py b/tests/unit/tokenizer/test_cohere_hf_tokenizer.py index f5f9671d..3ba43ea0 100644 --- a/tests/unit/tokenizer/test_cohere_hf_tokenizer.py +++ b/tests/unit/tokenizer/test_cohere_hf_tokenizer.py @@ -20,37 +20,29 @@ def tokenizer(): @staticmethod @pytest.fixture def expected_tokens(text): - return [ - 'string', - 'Ġwith', - 'Ġspecial', - 'Ġcharacters', - 'Ġlike', - 'Ġ!', - '@', - '#', - '$', - '%', - '^', - '&', - '*', - '()', - '_', - '+', - 'Ġæ', - 'Ĺ', - '¥', - 'æľ¬', - 'Ġspaces', - 'ĠĠĠ', - 'ĊĠ', - 'ĊĊ', - 'ĠCASE', - 'Ġc', - 'A', - 'se', - 'Ġ', - ] + return ['string', + 'Ġwith', + 'Ġspecial', + 'Ġcharacters', + 'Ġlike', + 'Ġ!', + '@', + '#$', + '%^', + '&', + '*', + '()', + '_', + '+', + 'ĠæĹ¥æľ¬', + 'Ġspaces', + 'ĠĠĠ', + 'ĊĠĊĊ', + 'ĠCASE', + 'Ġc', + 'A', + 'se', + 'Ġ'] @staticmethod def test_messages_token_count(tokenizer):