add cmmmu (EvolvingLMMs-Lab#44)

Co-authored-by: ygjin11 <[email protected]>

lmms_eval/tasks/cmmmu/cmmmu_test.yaml

@@ -0,0 +1,19 @@
+dataset_path: lmms-lab/CMMMU
+task: "cmmmu_test"
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.cmmmu_doc_to_visual
+doc_to_text: !function utils.cmmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.cmmmu_process_test_results_for_submission
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+  image_aspect_ratio: original
+metric_list:
+  - metric: cmmmu_acc
+    aggregation: !function utils.cmmmu_test_aggregate_results_for_submission
+    higher_is_better: true
+metadata:
+  - version: 0.0

lmms_eval/tasks/cmmmu/cmmmu_val.yaml

@@ -0,0 +1,19 @@
+dataset_path: lmms-lab/CMMMU
+task: "cmmmu_val"
+test_split: val
+output_type: generate_until
+doc_to_visual: !function utils.cmmmu_doc_to_visual
+doc_to_text: !function utils.cmmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.cmmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+  image_aspect_ratio: original
+metric_list:
+  - metric: cmmmu_acc
+    aggregation: !function utils.cmmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0