From e2686e8f64b4f6fe3dcc465ca5807e1a00a8d176 Mon Sep 17 00:00:00 2001 From: kcz358 <92624596+kcz358@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:50:37 +0800 Subject: [PATCH] Fix seedbench choices bugs (#45) --- lmms_eval/tasks/seedbench_2/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lmms_eval/tasks/seedbench_2/utils.py b/lmms_eval/tasks/seedbench_2/utils.py index 0f78cfe2..5ec7fccc 100644 --- a/lmms_eval/tasks/seedbench_2/utils.py +++ b/lmms_eval/tasks/seedbench_2/utils.py @@ -7,14 +7,15 @@ def seed_doc_to_visual(doc): def parse_choice_img(choice : str, img_token : str): if "jpg" in choice or "png" in choice: return img_token + return choice def seed_doc_to_text(doc, model_specific_kwargs = None): question = doc["question"] question.replace("", model_specific_kwargs['img_token']) question += "\n" + f"A. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n" - question += f"B. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n" - question += f"C. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n" - question += f"D. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}" + question += f"B. {parse_choice_img(doc['choice_b'], model_specific_kwargs['img_token'])}\n" + question += f"C. {parse_choice_img(doc['choice_c'], model_specific_kwargs['img_token'])}\n" + question += f"D. {parse_choice_img(doc['choice_d'], model_specific_kwargs['img_token'])}" if (doc['data_type'] == "Image Generation"): num_img_in_question = len(doc['image']) - 4 prepend_tokens = [model_specific_kwargs['img_token']] * num_img_in_question