Merge pull request #240 from EvolvingLMMs-Lab/replace-default-config-…

…to-non-yaml-format-to-avoid-loading-warning Update LMMS evaluation tasks for various subjects
EvolvingLMMs-Lab · Sep 10, 2024 · 9515cb4 · 9515cb4
2 parents 1618dc7 + b06e77e
commit 9515cb4
Show file tree

Hide file tree

Showing 33 changed files with 31 additions and 31 deletions.
diff --git a/...-bench-in-the-wild/_default_template.yaml → ...-bench-in-the-wild/_default_template_yaml b/...-bench-in-the-wild/_default_template.yaml → ...-bench-in-the-wild/_default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/arabic_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/arabic_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: arabic
     token: True
 task: "llava_in_the_wild_arabic"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/bengali_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/bengali_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: bengali
     token: True
 task: "llava_in_the_wild_bengali"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/chinese_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/chinese_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: chinese
     token: True
 task: "llava_in_the_wild_chinese"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/french_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/french_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: french
     token: True
 task: "llava_in_the_wild_french"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/hindi_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/hindi_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: hindi
     token: True
 task: "llava_in_the_wild_hindi"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/japanese_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/japanese_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: japanese
     token: True
 task: "llava_in_the_wild_japanese"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/russian_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/russian_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: russian
     token: True
 task: "llava_in_the_wild_russian"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/spanish_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/spanish_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
   config: spanish
   token: True
 task: "llava_in_the_wild_spanish"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/urdu_llava_in_the_wild.yaml b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/urdu_llava_in_the_wild.yaml
@@ -3,4 +3,4 @@ dataset_kwargs:
     config: urdu
     token: True
 task: "llava_in_the_wild_urdu"
-include: _default_template.yaml
+include: _default_template_yaml
diff --git a/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py b/lmms_eval/tasks/multilingual-llava-bench-in-the-wild/utils.py
@@ -17,7 +17,7 @@
 
 rule_dict = json.load(open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "rule.json"), "r"))
 
-with open(Path(__file__).parent / "_default_template.yaml", "r") as f:
+with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
     raw_data = f.readlines()
     safe_data = []
     for i, line in enumerate(raw_data):

diff --git a/...eval/tasks/mvbench/_default_template.yaml → ...eval/tasks/mvbench/_default_template_yaml b/...eval/tasks/mvbench/_default_template.yaml → ...eval/tasks/mvbench/_default_template_yaml
diff --git a/lmms_eval/tasks/mvbench/mvbench_action_antonym.yaml b/lmms_eval/tasks/mvbench/mvbench_action_antonym.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_action_antonym
 dataset_name: action_antonym
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_action_count.yaml b/lmms_eval/tasks/mvbench/mvbench_action_count.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_action_count
 dataset_name: action_count
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_action_localization.yaml b/lmms_eval/tasks/mvbench/mvbench_action_localization.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_action_localization
 dataset_name: action_localization
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_action_prediction.yaml b/lmms_eval/tasks/mvbench/mvbench_action_prediction.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_action_prediction
 dataset_name: action_prediction
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_action_sequence.yaml b/lmms_eval/tasks/mvbench/mvbench_action_sequence.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_action_sequence
 dataset_name: action_sequence
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_character_order.yaml b/lmms_eval/tasks/mvbench/mvbench_character_order.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_character_order
 dataset_name: character_order
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_counterfactual_inference.yaml b/lmms_eval/tasks/mvbench/mvbench_counterfactual_inference.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_counterfactual_inference
 dataset_name: counterfactual_inference
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_egocentric_navigation.yaml b/lmms_eval/tasks/mvbench/mvbench_egocentric_navigation.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_egocentric_navigation
 dataset_name: egocentric_navigation
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml b/lmms_eval/tasks/mvbench/mvbench_episodic_reasoning.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_episodic_reasoning
 dataset_name: episodic_reasoning
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_fine_grained_action.yaml b/lmms_eval/tasks/mvbench/mvbench_fine_grained_action.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_fine_grained_action
 dataset_name: fine_grained_action
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_fine_grained_pose.yaml b/lmms_eval/tasks/mvbench/mvbench_fine_grained_pose.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_fine_grained_pose
 dataset_name: fine_grained_pose
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_moving_attribute.yaml b/lmms_eval/tasks/mvbench/mvbench_moving_attribute.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_moving_attribute
 dataset_name: moving_attribute
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_moving_count.yaml b/lmms_eval/tasks/mvbench/mvbench_moving_count.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_moving_count
 dataset_name: moving_count
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_moving_direction.yaml b/lmms_eval/tasks/mvbench/mvbench_moving_direction.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_moving_direction
 dataset_name: moving_direction
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_object_existence.yaml b/lmms_eval/tasks/mvbench/mvbench_object_existence.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_object_existence
 dataset_name: object_existence
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_object_interaction.yaml b/lmms_eval/tasks/mvbench/mvbench_object_interaction.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_object_interaction
 dataset_name: object_interaction
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_object_shuffle.yaml b/lmms_eval/tasks/mvbench/mvbench_object_shuffle.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_object_shuffle
 dataset_name: object_shuffle
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_scene_transition.yaml b/lmms_eval/tasks/mvbench/mvbench_scene_transition.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_scene_transition
 dataset_name: scene_transition
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_state_change.yaml b/lmms_eval/tasks/mvbench/mvbench_state_change.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_state_change
 dataset_name: state_change
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/mvbench_unexpected_action.yaml b/lmms_eval/tasks/mvbench/mvbench_unexpected_action.yaml
@@ -1,4 +1,4 @@
-include: _default_template.yaml
+include: _default_template_yaml
 task: mvbench_unexpected_action
 dataset_name: unexpected_action
 test_split: train

diff --git a/lmms_eval/tasks/mvbench/utils.py b/lmms_eval/tasks/mvbench/utils.py
@@ -41,7 +41,7 @@
 hf_home = os.getenv("HF_HOME", "./~/.cache/huggingface")
 base_cache_dir = os.path.expanduser(hf_home)
 
-with open(Path(__file__).parent / "_default_template.yaml", "r") as f:
+with open(Path(__file__).parent / "_default_template_yaml", "r") as f:
     raw_data = f.readlines()
     safe_data = []
     for i, line in enumerate(raw_data):