Fix vLLM generation with sampling params

pyproject.toml

@@ -95,7 +95,7 @@ tensorboardX = ["tensorboardX"]
 vllm = ["vllm>=0.7.0", "ray", "more_itertools"]
 quality = ["ruff==v0.2.2","pre-commit"]
 tests = ["pytest==7.4.0"]
-dev = ["lighteval[accelerate,quality,tests,multilingual,math]"]
+dev = ["lighteval[accelerate,quality,tests,multilingual,math,extended_tasks]"]
 docs = ["hf-doc-builder", "watchdog"]
 extended_tasks = [
   "langdetect", # ifeval

src/lighteval/main_vllm.py

@@ -20,6 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 import os
+import re
 from typing import Optional
 
 from typer import Argument, Option
@@ -138,6 +139,8 @@ def vllm(
         generation_parameters = GenerationParameters.from_dict(config)
     else:
         generation_parameters = GenerationParameters.from_model_args(model_args)
+        # We slice out generation_parameters from model_args to avoid double-counting in the VLLMModelConfig
+        model_args = re.sub(r"generation_parameters=\{.*?\},?", "", model_args).strip(",")
         metric_options = {}
 
     model_args_dict: dict = {k.split("=")[0]: k.split("=")[1] if "=" in k else True for k in model_args.split(",")}

src/lighteval/tasks/extended/lcb/main.py

@@ -21,9 +21,8 @@
 # SOFTWARE.
 """Usage:
 lighteval vllm \
-    "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,tensor_parallel_size=4,max_model_length=32768,gpu_memory_utilisation=0.8" \
-    "extended|lcb:codegeneration|0|0" \
-    --custom-tasks src/lighteval/tasks/extended/lcb/main.py
+    "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,data_parallel_size=8,max_model_length=32768,gpu_memory_utilization=0.8,generation_parameters={temperature: 0.6,top_p: 0.95}" \
+    "extended|lcb:codegeneration|0|0"
 """
 
 import json