added changes from pr 245

huggingface · Nov 12, 2024 · 0346e15 · 0346e15
1 parent c345c73
commit 0346e15
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py
@@ -497,6 +497,8 @@ def from_pb(
 
         max_new_tokens = max(r.stopping_criteria.max_new_tokens for r in requests)
         max_input_length = max_truncation
+        if max_input_length < PAD_SEQUENCE_TO_MULTIPLE_OF:
+             max_input_length = PAD_SEQUENCE_TO_MULTIPLE_OF
         # TODO: by tokenizing all inputs at once we loose information on actual input lengths
         # this means that we cannot shift inputs to the left after a long input sequence
         # was filtered out