File tree 2 files changed +7
-10
lines changed
2 files changed +7
-10
lines changed Original file line number Diff line number Diff line change @@ -72,11 +72,10 @@ def to_tokens(
72
72
sides or prepend_bos.
73
73
Args:
74
74
input: List[str]]: The input to tokenize.
75
- move_to_device (bool): Whether to move the output tensor of tokens to the device the
76
- model lives on. Defaults to True
75
+ move_to_device (bool): Whether to move the output tensor of tokens to the device the model lives on. Defaults to True
77
76
truncate (bool): If the output tokens are too long, whether to truncate the output
78
- tokens to the model's max context window. Does nothing for shorter inputs.
79
- Defaults to True.
77
+ tokens to the model's max context window. Does nothing for shorter inputs. Defaults to
78
+ True.
80
79
"""
81
80
82
81
if len (input ) != 2 :
@@ -143,8 +142,7 @@ def forward(
143
142
144
143
Args:
145
144
input: The input to process. Can be one of:
146
- - List[str]: A list of two strings representing the two sentences NSP
147
- should be performed on
145
+ - List[str]: A list of two strings representing the two sentences NSP should be performed on
148
146
- torch.Tensor: Input tokens as integers with shape (batch, position)
149
147
return_type: Optional[str]: The type of output to return. Can be one of:
150
148
- None: Return nothing, don't calculate logits
Original file line number Diff line number Diff line change @@ -102,11 +102,10 @@ def to_tokens(
102
102
sides or prepend_bos.
103
103
Args:
104
104
input (Union[str, List[str]]): The input to tokenize.
105
- move_to_device (bool): Whether to move the output tensor of tokens to the device the
106
- model lives on. Defaults to True
105
+ move_to_device (bool): Whether to move the output tensor of tokens to the device the model lives on. Defaults to True
107
106
truncate (bool): If the output tokens are too long, whether to truncate the output
108
- tokens to the model's max context window. Does nothing for shorter inputs.
109
- Defaults to True.
107
+ tokens to the model's max context window. Does nothing for shorter inputs. Defaults to
108
+ True.
110
109
"""
111
110
112
111
assert self .tokenizer is not None , "Cannot use to_tokens without a tokenizer"
You can’t perform that action at this time.
0 commit comments