add mmmu (EvolvingLMMs-Lab#15)

* add mmme * black
kangreen0210 · Jan 24, 2024 · 5039565 · 5039565
1 parent bbcee02
commit 5039565
Show file tree

Hide file tree

Showing 7 changed files with 436 additions and 45 deletions.
diff --git a/lmms_eval/tasks/mmmu/_default_template_yaml b/lmms_eval/tasks/mmmu/_default_template_yaml
diff --git a/lmms_eval/tasks/mmmu/_mmmu.yaml b/lmms_eval/tasks/mmmu/_mmmu.yaml
diff --git a/lmms_eval/tasks/mmmu/mmmu.yaml b/lmms_eval/tasks/mmmu/mmmu.yaml
@@ -0,0 +1,19 @@
+dataset_path: lmms-lab/MMMU
+task: "mmmu"
+test_split: validation
+output_type: generate_until
+doc_to_visual: !function utils.mmmu_doc_to_visual
+doc_to_text: !function utils.mmmu_doc_to_text
+doc_to_target: "answer"
+# The return value of process_results will be used by metrics
+process_results: !function utils.mmmu_process_results
+# Note that the metric name can be either a registed metric function (such as the case for GQA) or a key name returned by process_results
+generation_kwargs:
+  max_new_tokens: 16
+  image_aspect_ratio: original
+metric_list:
+  - metric: mmmu_acc
+    aggregation: !function utils.mmmu_aggregate_results
+    higher_is_better: true
+metadata:
+  - version: 0.0
diff --git a/lmms_eval/tasks/mmmu/mmmu_accounting.yaml b/lmms_eval/tasks/mmmu/mmmu_accounting.yaml
diff --git a/lmms_eval/tasks/mmmu/mmmu_math.yaml b/lmms_eval/tasks/mmmu/mmmu_math.yaml