From ff0802ccf4afb97245c55eaf5b733780370f83a5 Mon Sep 17 00:00:00 2001 From: Pu Fanyi Date: Thu, 26 Sep 2024 12:28:03 +0800 Subject: [PATCH] [Fix] Fix the error when running models caused by `generate_until_multi_round` (#281) * fix * lint --- lmms_eval/models/batch_gpt4.py | 3 +++ lmms_eval/models/cambrian.py | 3 +++ lmms_eval/models/claude.py | 3 +++ lmms_eval/models/cogvlm2.py | 3 +++ lmms_eval/models/from_log.py | 3 +++ lmms_eval/models/fuyu.py | 3 +++ lmms_eval/models/gemini_api.py | 3 +++ lmms_eval/models/gpt4v.py | 3 +++ lmms_eval/models/idefics2.py | 3 +++ lmms_eval/models/instructblip.py | 3 +++ lmms_eval/models/internvl.py | 3 +++ lmms_eval/models/internvl2.py | 3 +++ lmms_eval/models/llama_vid.py | 3 +++ lmms_eval/models/llava.py | 3 +++ lmms_eval/models/llava_hf.py | 3 +++ lmms_eval/models/llava_sglang.py | 3 +++ lmms_eval/models/llava_vid.py | 3 +++ lmms_eval/models/longva.py | 3 +++ lmms_eval/models/mantis.py | 3 +++ lmms_eval/models/minicpm_v.py | 3 +++ lmms_eval/models/minimonkey.py | 3 +++ lmms_eval/models/mplug_owl_video.py | 3 +++ lmms_eval/models/phi3v.py | 3 +++ lmms_eval/models/qwen2_vl.py | 3 +++ lmms_eval/models/qwen_vl.py | 3 +++ lmms_eval/models/qwen_vl_api.py | 3 +++ lmms_eval/models/reka.py | 3 +++ lmms_eval/models/srt_api.py | 3 +++ lmms_eval/models/tinyllava.py | 3 +++ lmms_eval/models/video_chatgpt.py | 3 +++ lmms_eval/models/video_llava.py | 3 +++ lmms_eval/models/vila.py | 3 +++ lmms_eval/models/xcomposer2_4KHD.py | 3 +++ lmms_eval/models/xcomposer2d5.py | 3 +++ 34 files changed, 102 insertions(+) diff --git a/lmms_eval/models/batch_gpt4.py b/lmms_eval/models/batch_gpt4.py index fdc1a325..e624310a 100755 --- a/lmms_eval/models/batch_gpt4.py +++ b/lmms_eval/models/batch_gpt4.py @@ -202,3 +202,6 @@ def cancel_batch(self, batch_id): def list_batches(self, limit=10): return self.client.batches.list(limit=limit) + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for BatchGPT4") diff --git a/lmms_eval/models/cambrian.py b/lmms_eval/models/cambrian.py index 6bb395e5..855b87f9 100644 --- a/lmms_eval/models/cambrian.py +++ b/lmms_eval/models/cambrian.py @@ -308,3 +308,6 @@ def _collate(x): res = re_ords.get_original(res) pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Cambrian") diff --git a/lmms_eval/models/claude.py b/lmms_eval/models/claude.py index 823823dc..cc6af630 100644 --- a/lmms_eval/models/claude.py +++ b/lmms_eval/models/claude.py @@ -267,3 +267,6 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: assert False, "Not supported for claude" + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Claude") diff --git a/lmms_eval/models/cogvlm2.py b/lmms_eval/models/cogvlm2.py index ee346deb..ece7d01d 100644 --- a/lmms_eval/models/cogvlm2.py +++ b/lmms_eval/models/cogvlm2.py @@ -224,3 +224,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for CogVLM2") diff --git a/lmms_eval/models/from_log.py b/lmms_eval/models/from_log.py index 7de1f493..f9f88f91 100644 --- a/lmms_eval/models/from_log.py +++ b/lmms_eval/models/from_log.py @@ -114,3 +114,6 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: # TODO assert False, "not support" + + def generate_until_multi_round(self, requests) -> List[str]: + return generate_until(self, requests) diff --git a/lmms_eval/models/fuyu.py b/lmms_eval/models/fuyu.py index 73e66c15..6c9e0e0a 100755 --- a/lmms_eval/models/fuyu.py +++ b/lmms_eval/models/fuyu.py @@ -264,3 +264,6 @@ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=Non def tok_decode(self, tokens): return self.tokenizer.decode(tokens) + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Fuyu") diff --git a/lmms_eval/models/gemini_api.py b/lmms_eval/models/gemini_api.py index 7bebf129..dedafeed 100644 --- a/lmms_eval/models/gemini_api.py +++ b/lmms_eval/models/gemini_api.py @@ -181,6 +181,9 @@ def get_uuid(task, split, doc_id): pbar.close() return res + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Gemini API") + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: # TODO assert False, "Gemini API not support" diff --git a/lmms_eval/models/gpt4v.py b/lmms_eval/models/gpt4v.py index aac62a87..af313a57 100755 --- a/lmms_eval/models/gpt4v.py +++ b/lmms_eval/models/gpt4v.py @@ -230,6 +230,9 @@ def generate_until(self, requests) -> List[str]: pbar.close() return res + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for GPT4V") + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: # TODO assert False, "GPT4V not support" diff --git a/lmms_eval/models/idefics2.py b/lmms_eval/models/idefics2.py index 644ae41e..eac34992 100644 --- a/lmms_eval/models/idefics2.py +++ b/lmms_eval/models/idefics2.py @@ -229,3 +229,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Idefics2") diff --git a/lmms_eval/models/instructblip.py b/lmms_eval/models/instructblip.py index d07efd37..dc3b0926 100755 --- a/lmms_eval/models/instructblip.py +++ b/lmms_eval/models/instructblip.py @@ -225,3 +225,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for InstructBlip") diff --git a/lmms_eval/models/internvl.py b/lmms_eval/models/internvl.py index c238bf59..093aa605 100644 --- a/lmms_eval/models/internvl.py +++ b/lmms_eval/models/internvl.py @@ -491,3 +491,6 @@ def _collate(x): def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: pass + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for InternVL") diff --git a/lmms_eval/models/internvl2.py b/lmms_eval/models/internvl2.py index fadf778e..5de4ce1f 100644 --- a/lmms_eval/models/internvl2.py +++ b/lmms_eval/models/internvl2.py @@ -281,3 +281,6 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: assert False, "Not implemented yet." + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for InternVL2") diff --git a/lmms_eval/models/llama_vid.py b/lmms_eval/models/llama_vid.py index 8c1c56cb..60945186 100644 --- a/lmms_eval/models/llama_vid.py +++ b/lmms_eval/models/llama_vid.py @@ -277,3 +277,6 @@ def rank(self): @property def world_size(self): return self._world_size + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LLaMAVid") diff --git a/lmms_eval/models/llava.py b/lmms_eval/models/llava.py index 9940bc76..2de73cf5 100755 --- a/lmms_eval/models/llava.py +++ b/lmms_eval/models/llava.py @@ -425,3 +425,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LLaVA") diff --git a/lmms_eval/models/llava_hf.py b/lmms_eval/models/llava_hf.py index a2f30476..d83a15c0 100644 --- a/lmms_eval/models/llava_hf.py +++ b/lmms_eval/models/llava_hf.py @@ -387,3 +387,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF") diff --git a/lmms_eval/models/llava_sglang.py b/lmms_eval/models/llava_sglang.py index 233bbe9c..986de912 100644 --- a/lmms_eval/models/llava_sglang.py +++ b/lmms_eval/models/llava_sglang.py @@ -160,3 +160,6 @@ def prepare_arguments_parallel(contexts, batched_visuals, max_workers=64): pbar.close() runtime.shutdown() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LLaVA-SGLang") diff --git a/lmms_eval/models/llava_vid.py b/lmms_eval/models/llava_vid.py index e013ed8f..088a5619 100755 --- a/lmms_eval/models/llava_vid.py +++ b/lmms_eval/models/llava_vid.py @@ -433,3 +433,6 @@ def generate_until(self, requests) -> List[str]: res.append(outputs) pbar.update(1) return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LLaVAVid") diff --git a/lmms_eval/models/longva.py b/lmms_eval/models/longva.py index ebc7dde8..016d0ecb 100644 --- a/lmms_eval/models/longva.py +++ b/lmms_eval/models/longva.py @@ -471,3 +471,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for LongVA") diff --git a/lmms_eval/models/mantis.py b/lmms_eval/models/mantis.py index dc3b55f0..7ce9005a 100644 --- a/lmms_eval/models/mantis.py +++ b/lmms_eval/models/mantis.py @@ -307,3 +307,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation for Mantis") diff --git a/lmms_eval/models/minicpm_v.py b/lmms_eval/models/minicpm_v.py index 553c0819..962405bd 100755 --- a/lmms_eval/models/minicpm_v.py +++ b/lmms_eval/models/minicpm_v.py @@ -219,3 +219,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/minimonkey.py b/lmms_eval/models/minimonkey.py index 1ecd857c..7f3e2e7d 100644 --- a/lmms_eval/models/minimonkey.py +++ b/lmms_eval/models/minimonkey.py @@ -215,6 +215,9 @@ def _collate(x): pbar.close() return res + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") + import numpy as np import torchvision.transforms as T diff --git a/lmms_eval/models/mplug_owl_video.py b/lmms_eval/models/mplug_owl_video.py index f6d25d00..04676cdb 100644 --- a/lmms_eval/models/mplug_owl_video.py +++ b/lmms_eval/models/mplug_owl_video.py @@ -194,3 +194,6 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: return super().loglikelihood(requests) + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/phi3v.py b/lmms_eval/models/phi3v.py index 61e36386..9617f673 100644 --- a/lmms_eval/models/phi3v.py +++ b/lmms_eval/models/phi3v.py @@ -215,3 +215,6 @@ def _collate(x): res = re_ords.get_original(res) pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/qwen2_vl.py b/lmms_eval/models/qwen2_vl.py index 4705a6a3..9848f911 100755 --- a/lmms_eval/models/qwen2_vl.py +++ b/lmms_eval/models/qwen2_vl.py @@ -273,3 +273,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/qwen_vl.py b/lmms_eval/models/qwen_vl.py index 79adc532..855a7c30 100755 --- a/lmms_eval/models/qwen_vl.py +++ b/lmms_eval/models/qwen_vl.py @@ -307,3 +307,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/qwen_vl_api.py b/lmms_eval/models/qwen_vl_api.py index 47a25a15..7720cd55 100644 --- a/lmms_eval/models/qwen_vl_api.py +++ b/lmms_eval/models/qwen_vl_api.py @@ -121,3 +121,6 @@ def flatten(self, input): for j in i: new_list.append(j) return new_list + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/reka.py b/lmms_eval/models/reka.py index 744f54ee..dccd7665 100644 --- a/lmms_eval/models/reka.py +++ b/lmms_eval/models/reka.py @@ -193,3 +193,6 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: # TODO assert False, "Reka not support loglikelihood" + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/srt_api.py b/lmms_eval/models/srt_api.py index 9752bd96..cefd6feb 100755 --- a/lmms_eval/models/srt_api.py +++ b/lmms_eval/models/srt_api.py @@ -293,3 +293,6 @@ async def _process(request): def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: # TODO assert False, "GPT4V not support" + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/tinyllava.py b/lmms_eval/models/tinyllava.py index 3b0ff7cb..2aaa4b19 100755 --- a/lmms_eval/models/tinyllava.py +++ b/lmms_eval/models/tinyllava.py @@ -408,3 +408,6 @@ def _collate(x): pbar.close() return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/video_chatgpt.py b/lmms_eval/models/video_chatgpt.py index bd1017c2..0a86b074 100644 --- a/lmms_eval/models/video_chatgpt.py +++ b/lmms_eval/models/video_chatgpt.py @@ -203,3 +203,6 @@ def rank(self): @property def world_size(self): return self._world_size + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/video_llava.py b/lmms_eval/models/video_llava.py index e2e4f1c9..eeddd9a0 100644 --- a/lmms_eval/models/video_llava.py +++ b/lmms_eval/models/video_llava.py @@ -209,3 +209,6 @@ def generate_until(self, requests) -> List[str]: res.append(outputs) pbar.update(1) return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/vila.py b/lmms_eval/models/vila.py index 44b67056..ca4a0304 100755 --- a/lmms_eval/models/vila.py +++ b/lmms_eval/models/vila.py @@ -366,3 +366,6 @@ def generate_until(self, requests) -> List[str]: res.append(outputs) pbar.update(1) return res + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") diff --git a/lmms_eval/models/xcomposer2_4KHD.py b/lmms_eval/models/xcomposer2_4KHD.py index 375ba9d0..7a5235a3 100644 --- a/lmms_eval/models/xcomposer2_4KHD.py +++ b/lmms_eval/models/xcomposer2_4KHD.py @@ -229,6 +229,9 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: return super().loglikelihood(requests) + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation") + def padding_336(b): width, height = b.size diff --git a/lmms_eval/models/xcomposer2d5.py b/lmms_eval/models/xcomposer2d5.py index f5d4dfe0..0662d1f8 100644 --- a/lmms_eval/models/xcomposer2d5.py +++ b/lmms_eval/models/xcomposer2d5.py @@ -182,3 +182,6 @@ def generate_until(self, requests) -> List[str]: def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: assert False, "Not implemented yet." + + def generate_until_multi_round(self, requests) -> List[str]: + raise NotImplementedError("TODO: Implement multi-round generation")