From 376737883fa75967e1a66f7aac2b567046447172 Mon Sep 17 00:00:00 2001
From: Lewis Tunstall <lewis.c.tunstall@gmail.com>
Date: Thu, 20 Feb 2025 21:31:33 +0000
Subject: [PATCH] Fix vLLM generation with sampling params

---
 pyproject.toml                           | 2 +-
 src/lighteval/main_vllm.py               | 3 +++
 src/lighteval/tasks/extended/lcb/main.py | 5 ++---
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 721ad1d5..64f6d0a6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -95,7 +95,7 @@ tensorboardX = ["tensorboardX"]
 vllm = ["vllm>=0.7.0", "ray", "more_itertools"]
 quality = ["ruff==v0.2.2","pre-commit"]
 tests = ["pytest==7.4.0"]
-dev = ["lighteval[accelerate,quality,tests,multilingual,math]"]
+dev = ["lighteval[accelerate,quality,tests,multilingual,math,extended_tasks]"]
 docs = ["hf-doc-builder", "watchdog"]
 extended_tasks = [
   "langdetect", # ifeval
diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py
index 8750cd47..4ffcd60e 100644
--- a/src/lighteval/main_vllm.py
+++ b/src/lighteval/main_vllm.py
@@ -20,6 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 import os
+import re
 from typing import Optional
 
 from typer import Argument, Option
@@ -138,6 +139,8 @@ def vllm(
         generation_parameters = GenerationParameters.from_dict(config)
     else:
         generation_parameters = GenerationParameters.from_model_args(model_args)
+        # We slice out generation_parameters from model_args to avoid double-counting in the VLLMModelConfig
+        model_args = re.sub(r"generation_parameters=\{.*?\},?", "", model_args).strip(",")
         metric_options = {}
 
     model_args_dict: dict = {k.split("=")[0]: k.split("=")[1] if "=" in k else True for k in model_args.split(",")}
diff --git a/src/lighteval/tasks/extended/lcb/main.py b/src/lighteval/tasks/extended/lcb/main.py
index b3bcd3c9..ff3d16dc 100644
--- a/src/lighteval/tasks/extended/lcb/main.py
+++ b/src/lighteval/tasks/extended/lcb/main.py
@@ -21,9 +21,8 @@
 # SOFTWARE.
 """Usage:
 lighteval vllm \
-    "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,tensor_parallel_size=4,max_model_length=32768,gpu_memory_utilisation=0.8" \
-    "extended|lcb:codegeneration|0|0" \
-    --custom-tasks src/lighteval/tasks/extended/lcb/main.py
+    "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,data_parallel_size=8,max_model_length=32768,gpu_memory_utilization=0.8,generation_parameters={temperature: 0.6,top_p: 0.95}" \
+    "extended|lcb:codegeneration|0|0"
 """
 
 import json