Fix seedbench choices bugs (EvolvingLMMs-Lab#45)

kangreen0210 · Feb 13, 2024 · e2686e8 · e2686e8
1 parent bf93c62
commit e2686e8
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/lmms_eval/tasks/seedbench_2/utils.py b/lmms_eval/tasks/seedbench_2/utils.py
@@ -7,14 +7,15 @@ def seed_doc_to_visual(doc):
 def parse_choice_img(choice : str, img_token : str):
     if "jpg" in choice or "png" in choice:
         return img_token
+    return choice
 
 def seed_doc_to_text(doc, model_specific_kwargs = None):
     question = doc["question"]
     question.replace("<img>", model_specific_kwargs['img_token'])
     question += "\n" + f"A. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n"
-    question += f"B. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n"
-    question += f"C. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}\n"
-    question += f"D. {parse_choice_img(doc['choice_a'], model_specific_kwargs['img_token'])}"
+    question += f"B. {parse_choice_img(doc['choice_b'], model_specific_kwargs['img_token'])}\n"
+    question += f"C. {parse_choice_img(doc['choice_c'], model_specific_kwargs['img_token'])}\n"
+    question += f"D. {parse_choice_img(doc['choice_d'], model_specific_kwargs['img_token'])}"
     if (doc['data_type'] == "Image Generation"):
         num_img_in_question = len(doc['image']) - 4
         prepend_tokens = [model_specific_kwargs['img_token']] * num_img_in_question