diff --git a/src/mistral_common/tokens/tokenizers/tekken.py b/src/mistral_common/tokens/tokenizers/tekken.py index 9197a09..07b5f71 100644 --- a/src/mistral_common/tokens/tokenizers/tekken.py +++ b/src/mistral_common/tokens/tokenizers/tekken.py @@ -236,7 +236,7 @@ def _decode_all(self, tokens: List[int], special_token_policy: SpecialTokenPolic "```\nfrom mistral_common.tokens.tokenizers.mistral import MistralTokenizer" "\nfrom mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy" "\n\ntokenizer = MistralTokenizer.v3(is_tekken=True)" - "\ntokenizer.special_token_policy = SpecialTokenPolicy.IGNORE # or SpecialTokenPolicy.KEEP" + "\ntokenizer.instruct_tokenizer.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE # or SpecialTokenPolicy.KEEP" "\n```" ) elif special_token_policy == SpecialTokenPolicy.KEEP: