Skip to content

Commit

Permalink
Up
Browse files Browse the repository at this point in the history
  • Loading branch information
patrickvonplaten committed Sep 19, 2024
1 parent e678136 commit a927bc3
Showing 1 changed file with 2 additions and 1 deletion.
3 changes: 2 additions & 1 deletion src/mistral_common/tokens/tokenizers/tekken.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,8 @@ def _decode_all(self, tokens: List[int], special_token_policy: SpecialTokenPolic
"```\nfrom mistral_common.tokens.tokenizers.mistral import MistralTokenizer"
"\nfrom mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy"
"\n\ntokenizer = MistralTokenizer.v3(is_tekken=True)"
"\ntokenizer.instruct_tokenizer.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE # or SpecialTokenPolicy.KEEP"
"\ntekken = tokenizer.instruct_tokenizer.tokenizer"
"\ntekken.special_token_policy = SpecialTokenPolicy.IGNORE # or SpecialTokenPolicy.KEEP"
"\n```"
)
elif special_token_policy == SpecialTokenPolicy.KEEP:
Expand Down

0 comments on commit a927bc3

Please sign in to comment.