1.4.2 release

mistralai · Sep 19, 2024 · e678136 · e678136
1 parent 992f4a0
commit e678136
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/mistral_common/tokens/tokenizers/tekken.py b/src/mistral_common/tokens/tokenizers/tekken.py
@@ -236,7 +236,7 @@ def _decode_all(self, tokens: List[int], special_token_policy: SpecialTokenPolic
                         "```\nfrom mistral_common.tokens.tokenizers.mistral import MistralTokenizer"
                         "\nfrom mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy"
                         "\n\ntokenizer = MistralTokenizer.v3(is_tekken=True)"
-                        "\ntokenizer.special_token_policy = SpecialTokenPolicy.IGNORE  # or SpecialTokenPolicy.KEEP"
+                        "\ntokenizer.instruct_tokenizer.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE  # or SpecialTokenPolicy.KEEP"
                         "\n```"
                     )
                 elif special_token_policy == SpecialTokenPolicy.KEEP: