add jmmmu (#286)

* add jmmmu * Apply black formatting to utils.py * change the location of jmmmu
EvolvingLMMs-Lab · Sep 30, 2024 · 7c2d91c · 7c2d91c
1 parent 0f1576e
commit 7c2d91c
Show file tree

Hide file tree

Showing 32 changed files with 614 additions and 0 deletions.
diff --git a/docs/current_tasks.md b/docs/current_tasks.md
@@ -162,6 +162,7 @@
 - [ICON-QA](https://iconqa.github.io/) (iconqa)
   - ICON-QA Validation (iconqa_val)
   - ICON-QA Test (iconqa_test)
+- [JMMMU](https://mmmu-japanese-benchmark.github.io/JMMMU/) (jmmmu)
 - [LLaVA-NeXT-Interleave-Bench](https://huggingface.co/datasets/lmms-lab/LLaVA-NeXT-Interleave-Bench) (llava_interleave_bench)
   - llava_interleave_bench_in_domain
   - llava_interleave_bench_out_domain

diff --git a/lmms_eval/tasks/jmmmu/_default_template_yaml b/lmms_eval/tasks/jmmmu/_default_template_yaml
@@ -0,0 +1,20 @@
+dataset_path: JMMMU/JMMMU
+test_split: test
+output_type: generate_until
+doc_to_visual: !function utils.jmmmu_doc_to_visual
+doc_to_text: !function utils.jmmmu_doc_to_text
+doc_to_target: "answer"
+
+process_results: !function utils.jmmmu_process_results
+
+metric_list:
+  - metric: jmmmu_acc
+    aggregation: !function utils.jmmmu_aggregate_results
+    higher_is_better: true
+
+generation_kwargs:
+  max_new_tokens: 1024
+
+metadata:
+  version: 0.0
+  interleaved_format: false
diff --git a/lmms_eval/tasks/jmmmu/jmmmu.yaml b/lmms_eval/tasks/jmmmu/jmmmu.yaml
@@ -0,0 +1,18 @@
+group: jmmmu
+group_alias: jmmmu_all
+task:
+  - group: culture_agnostic
+    task:
+      - jmmmu_culture_agnostic
+    aggregate_metric_list:
+      - metric: jmmmu_acc
+        weight_by_size: True
+  - group: culture_specific
+    task:
+      - jmmmu_culture_specific
+    aggregate_metric_list:
+      - metric: jmmmu_acc
+        weight_by_size: True
+aggregate_metric_list:
+  - metric: jmmmu_acc
+    weight_by_size: True
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_accounting.yaml b/lmms_eval/tasks/jmmmu/jmmmu_accounting.yaml
@@ -0,0 +1,4 @@
+dataset_name: Accounting
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_accounting"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_agriculture.yaml b/lmms_eval/tasks/jmmmu/jmmmu_agriculture.yaml
@@ -0,0 +1,4 @@
+dataset_name: Agriculture
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_agriculture"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_architecture_and_engineering.yaml b/lmms_eval/tasks/jmmmu/jmmmu_architecture_and_engineering.yaml
@@ -0,0 +1,4 @@
+dataset_name: Architecture_and_Engineering
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_architecture_and_engineering"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_basic_medical_science.yaml b/lmms_eval/tasks/jmmmu/jmmmu_basic_medical_science.yaml
@@ -0,0 +1,4 @@
+dataset_name: Basic_Medical_Science
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_basic_medical_science"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_biology.yaml b/lmms_eval/tasks/jmmmu/jmmmu_biology.yaml
@@ -0,0 +1,4 @@
+dataset_name: Biology
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_biology"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_chemistry.yaml b/lmms_eval/tasks/jmmmu/jmmmu_chemistry.yaml
@@ -0,0 +1,4 @@
+dataset_name: Chemistry
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_chemistry"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_clinical_medicine.yaml b/lmms_eval/tasks/jmmmu/jmmmu_clinical_medicine.yaml
@@ -0,0 +1,4 @@
+dataset_name: Clinical_Medicine
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_clinical_medicine"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_computer_science.yaml b/lmms_eval/tasks/jmmmu/jmmmu_computer_science.yaml
@@ -0,0 +1,4 @@
+dataset_name: Computer_Science
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_computer_science"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_design.yaml b/lmms_eval/tasks/jmmmu/jmmmu_design.yaml
@@ -0,0 +1,4 @@
+dataset_name: Design
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_design"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_diagnostics_and_laboratory_medicine.yaml b/lmms_eval/tasks/jmmmu/jmmmu_diagnostics_and_laboratory_medicine.yaml
@@ -0,0 +1,4 @@
+dataset_name: Diagnostics_and_Laboratory_Medicine
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_diagnostics_and_laboratory_medicine"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_economics.yaml b/lmms_eval/tasks/jmmmu/jmmmu_economics.yaml
@@ -0,0 +1,4 @@
+dataset_name: Economics
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_economics"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_electronics.yaml b/lmms_eval/tasks/jmmmu/jmmmu_electronics.yaml
@@ -0,0 +1,4 @@
+dataset_name: Electronics
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_electronics"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_energy_and_power.yaml b/lmms_eval/tasks/jmmmu/jmmmu_energy_and_power.yaml
@@ -0,0 +1,4 @@
+dataset_name: Energy_and_Power
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_energy_and_power"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_finance.yaml b/lmms_eval/tasks/jmmmu/jmmmu_finance.yaml
@@ -0,0 +1,5 @@
+
+dataset_name: Finance
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_finance"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_japanese_art.yaml b/lmms_eval/tasks/jmmmu/jmmmu_japanese_art.yaml
@@ -0,0 +1,4 @@
+dataset_name: Japanese_Art
+tag: "jmmmu_culture_specific"
+task: "jmmmu_japanese_art"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_japanese_heritage.yaml b/lmms_eval/tasks/jmmmu/jmmmu_japanese_heritage.yaml
@@ -0,0 +1,4 @@
+dataset_name: Japanese_Heritage
+tag: "jmmmu_culture_specific"
+task: "jmmmu_japanese_heritage"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_japanese_history.yaml b/lmms_eval/tasks/jmmmu/jmmmu_japanese_history.yaml
@@ -0,0 +1,4 @@
+dataset_name: Japanese_History
+tag: "jmmmu_culture_specific"
+task: "jmmmu_japanese_history"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_manage.yaml b/lmms_eval/tasks/jmmmu/jmmmu_manage.yaml
@@ -0,0 +1,4 @@
+dataset_name: Manage
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_manage"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_marketing.yaml b/lmms_eval/tasks/jmmmu/jmmmu_marketing.yaml
@@ -0,0 +1,4 @@
+dataset_name: Marketing
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_marketing"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_materials.yaml b/lmms_eval/tasks/jmmmu/jmmmu_materials.yaml
@@ -0,0 +1,4 @@
+dataset_name: Materials
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_materials"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_math.yaml b/lmms_eval/tasks/jmmmu/jmmmu_math.yaml
@@ -0,0 +1,4 @@
+dataset_name: Math
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_math"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_mechanical_engineering.yaml b/lmms_eval/tasks/jmmmu/jmmmu_mechanical_engineering.yaml
@@ -0,0 +1,4 @@
+dataset_name: Mechanical_Engineering
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_mechanical_engineering"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_music.yaml b/lmms_eval/tasks/jmmmu/jmmmu_music.yaml
@@ -0,0 +1,4 @@
+dataset_name: Music
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_music"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_pharmacy.yaml b/lmms_eval/tasks/jmmmu/jmmmu_pharmacy.yaml
@@ -0,0 +1,4 @@
+dataset_name: Pharmacy
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_pharmacy"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_physics.yaml b/lmms_eval/tasks/jmmmu/jmmmu_physics.yaml
@@ -0,0 +1,4 @@
+dataset_name: Physics
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_physics"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_psychology.yaml b/lmms_eval/tasks/jmmmu/jmmmu_psychology.yaml
@@ -0,0 +1,4 @@
+dataset_name: Psychology
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_psychology"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_public_health.yaml b/lmms_eval/tasks/jmmmu/jmmmu_public_health.yaml
@@ -0,0 +1,4 @@
+dataset_name: Public_Health
+tag: "jmmmu_culture_agnostic"
+task: "jmmmu_public_health"
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/jmmmu/jmmmu_world_history.yaml b/lmms_eval/tasks/jmmmu/jmmmu_world_history.yaml
@@ -0,0 +1,4 @@
+dataset_name: World_History
+tag: "jmmmu_culture_specific"
+task: "jmmmu_world_history"
+include: _default_template_yaml