From 3827381fc9b89fb0634dbeea37fef16bb2f5ecdd Mon Sep 17 00:00:00 2001 From: Fanyi Pu Date: Fri, 6 Sep 2024 16:31:58 +0800 Subject: [PATCH 1/4] fix font path --- lmms_eval/tasks/mmmu/utils_group_img.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lmms_eval/tasks/mmmu/utils_group_img.py b/lmms_eval/tasks/mmmu/utils_group_img.py index 7fd9fca2..d5d23936 100644 --- a/lmms_eval/tasks/mmmu/utils_group_img.py +++ b/lmms_eval/tasks/mmmu/utils_group_img.py @@ -18,7 +18,7 @@ def add_order_label(image, label, font_size=40): # Define font for the label # font_path = fm.findfont(fm.FontProperties(family=font_family)) - font_path = "./arial.ttf" + font_path = os.path.join(__file__, os.pardir, "arial.ttf") font = ImageFont.truetype(font_path, font_size) # Calculate text size and position From afd33384f4404b235f0734eaf9dd83e5f4f7ebc8 Mon Sep 17 00:00:00 2001 From: Fanyi Pu Date: Fri, 6 Sep 2024 17:44:11 +0800 Subject: [PATCH 2/4] deepcopy --- lmms_eval/api/task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index 7a8081ad..7d4c6da9 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -7,6 +7,7 @@ import random import re import shutil +import copy import subprocess from collections.abc import Callable from dataclasses import asdict, dataclass, field @@ -1245,7 +1246,7 @@ def construct_requests(self, doc_id: int, ctx: str, **kwargs) -> Union[List[Inst return request_list elif self.OUTPUT_TYPE == "generate_until": - arguments = (ctx, self.config.generation_kwargs, self.doc_to_visual, doc_id, self.config.task, split) + arguments = (ctx, copy.deepcopy(self.config.generation_kwargs), self.doc_to_visual, doc_id, self.config.task, split) return Instance(request_type=self.OUTPUT_TYPE, arguments=arguments, idx=0, **kwargs) # TODO: we add a full_docs interface here for some evaluations that needs to access the full datasets during process_results function. we may have better ways to handle this. From 790436258ee30eac8f9d2301462de7869b7ae7be Mon Sep 17 00:00:00 2001 From: Fanyi Pu Date: Fri, 6 Sep 2024 17:48:13 +0800 Subject: [PATCH 3/4] lint --- lmms_eval/api/samplers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lmms_eval/api/samplers.py b/lmms_eval/api/samplers.py index 2cecfe22..f77065e8 100755 --- a/lmms_eval/api/samplers.py +++ b/lmms_eval/api/samplers.py @@ -37,9 +37,7 @@ def get_context(self, doc, num_fewshot): + ( str(self.doc_to_target(doc)[0]) if type(self.doc_to_target(doc)) is list - else self.doc_to_target(doc) - if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) - else str(self.doc_to_choice(doc)[self.doc_to_target(doc)]) + else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)]) ) for doc in selected_docs ] From 3e2ed44790baf313d1904bec4376e775e570867f Mon Sep 17 00:00:00 2001 From: Fanyi Pu Date: Fri, 6 Sep 2024 17:52:38 +0800 Subject: [PATCH 4/4] lint --- lmms_eval/api/samplers.py | 4 +++- lmms_eval/api/task.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lmms_eval/api/samplers.py b/lmms_eval/api/samplers.py index f77065e8..2cecfe22 100755 --- a/lmms_eval/api/samplers.py +++ b/lmms_eval/api/samplers.py @@ -37,7 +37,9 @@ def get_context(self, doc, num_fewshot): + ( str(self.doc_to_target(doc)[0]) if type(self.doc_to_target(doc)) is list - else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)]) + else self.doc_to_target(doc) + if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) + else str(self.doc_to_choice(doc)[self.doc_to_target(doc)]) ) for doc in selected_docs ] diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index 7d4c6da9..8b312756 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -1,5 +1,6 @@ import abc import ast +import copy import inspect import itertools import json @@ -7,7 +8,6 @@ import random import re import shutil -import copy import subprocess from collections.abc import Callable from dataclasses import asdict, dataclass, field