Merge pull request #7 from EvolvingLMMs-Lab/pufanyi/mmbench

Fix mmbench dataset submission format
EvolvingLMMs-Lab · Mar 12, 2024 · ea6788b · ea6788b
2 parents cbb3874 + d18f114
commit ea6788b
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 6 deletions.
diff --git a/lmms_eval/api/samplers.py b/lmms_eval/api/samplers.py
@@ -37,9 +37,7 @@ def get_context(self, doc, num_fewshot):
                     + (
                         str(self.doc_to_target(doc)[0])
                         if type(self.doc_to_target(doc)) is list
-                        else self.doc_to_target(doc)
-                        if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str)
-                        else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
+                        else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
                     )
                     for doc in selected_docs
                 ]

diff --git a/lmms_eval/tasks/mmbench/cc_utils.py b/lmms_eval/tasks/mmbench/cc_utils.py
@@ -50,7 +50,7 @@ def mmbench_cn_cc_doc_to_text(doc, model_specific_prompt_kwargs=None):
 
 def mmbench_cn_cc_process_results(doc, results):
     model_response = results[0].strip()
-    return {
+    data = {
         "submission": {
             "index": doc["index"],
             "question": doc["question"],
@@ -60,6 +60,10 @@ def mmbench_cn_cc_process_results(doc, results):
             "category": doc["category"],
         }
     }
+    option_candidate = ["A", "B", "C", "D", "E"]
+    for c in option_candidate:
+        data["submission"][c] = doc.get(c, "nan")
+    return data
 
 
 def mmbench_cn_cc_aggregate_results(results):

diff --git a/lmms_eval/tasks/mmbench/cn_utils.py b/lmms_eval/tasks/mmbench/cn_utils.py
@@ -54,7 +54,7 @@ def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None):
 
 def mmbench_process_results(doc, results):
     model_response = results[0].strip()
-    return {
+    data = {
         "submission": {
             "index": doc["index"],
             "question": doc["question"],
@@ -67,6 +67,10 @@ def mmbench_process_results(doc, results):
             "L2-category": doc["L2-category"],
         }
     }
+    option_candidate = ["A", "B", "C", "D", "E"]
+    for c in option_candidate:
+        data["submission"][c] = doc.get(c, "nan")
+    return data
 
 
 def mmbench_aggregate_dev_results(results, args):

diff --git a/lmms_eval/tasks/mmbench/en_utils.py b/lmms_eval/tasks/mmbench/en_utils.py
@@ -54,7 +54,7 @@ def mmbench_doc_to_text(doc, model_specific_prompt_kwargs=None):
 
 def mmbench_process_results(doc, results):
     model_response = results[0].strip()
-    return {
+    data = {
         "submission": {
             "index": doc["index"],
             "question": doc["question"],
@@ -67,6 +67,10 @@ def mmbench_process_results(doc, results):
             "L2-category": doc["l2-category"],
         }
     }
+    option_candidate = ["A", "B", "C", "D", "E"]
+    for c in option_candidate:
+        data["submission"][c] = doc.get(c, "nan")
+    return data
 
 
 def mmbench_aggregate_dev_results(results, args):