[Fix] Fix the error when running models caused by `generate_until_mul…

…ti_round` (#281)

* fix

* lint

lmms_eval/models/batch_gpt4.py

@@ -202,3 +202,6 @@ def cancel_batch(self, batch_id):
 
     def list_batches(self, limit=10):
         return self.client.batches.list(limit=limit)
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for BatchGPT4")

lmms_eval/models/cambrian.py

@@ -308,3 +308,6 @@ def _collate(x):
         res = re_ords.get_original(res)
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for Cambrian")

lmms_eval/models/claude.py

@@ -267,3 +267,6 @@ def generate_until(self, requests) -> List[str]:
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         assert False, "Not supported for claude"
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for Claude")

lmms_eval/models/cogvlm2.py

@@ -224,3 +224,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for CogVLM2")

lmms_eval/models/from_log.py

@@ -114,3 +114,6 @@ def generate_until(self, requests) -> List[str]:
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         assert False, "not support"
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        return generate_until(self, requests)

lmms_eval/models/fuyu.py

@@ -264,3 +264,6 @@ def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=Non
 
     def tok_decode(self, tokens):
         return self.tokenizer.decode(tokens)
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for Fuyu")

lmms_eval/models/gemini_api.py

@@ -181,6 +181,9 @@ def get_uuid(task, split, doc_id):
         pbar.close()
         return res
 
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for Gemini API")
+
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         assert False, "Gemini API not support"

lmms_eval/models/gpt4v.py

@@ -230,6 +230,9 @@ def generate_until(self, requests) -> List[str]:
         pbar.close()
         return res
 
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for GPT4V")
+
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         assert False, "GPT4V not support"

lmms_eval/models/idefics2.py

@@ -229,3 +229,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for Idefics2")

lmms_eval/models/instructblip.py

@@ -225,3 +225,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for InstructBlip")

lmms_eval/models/internvl.py

@@ -491,3 +491,6 @@ def _collate(x):
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         pass
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for InternVL")

lmms_eval/models/internvl2.py

@@ -281,3 +281,6 @@ def generate_until(self, requests) -> List[str]:
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         assert False, "Not implemented yet."
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for InternVL2")

lmms_eval/models/llama_vid.py

@@ -277,3 +277,6 @@ def rank(self):
     @property
     def world_size(self):
         return self._world_size
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaMAVid")

lmms_eval/models/llava.py

@@ -425,3 +425,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaVA")

lmms_eval/models/llava_hf.py

@@ -387,3 +387,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaVAHF")

lmms_eval/models/llava_sglang.py

@@ -160,3 +160,6 @@ def prepare_arguments_parallel(contexts, batched_visuals, max_workers=64):
         pbar.close()
         runtime.shutdown()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaVA-SGLang")

lmms_eval/models/llava_vid.py

@@ -433,3 +433,6 @@ def generate_until(self, requests) -> List[str]:
             res.append(outputs)
             pbar.update(1)
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LLaVAVid")

lmms_eval/models/longva.py

@@ -471,3 +471,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for LongVA")

lmms_eval/models/mantis.py

@@ -307,3 +307,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation for Mantis")

lmms_eval/models/minicpm_v.py

@@ -219,3 +219,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/minimonkey.py

@@ -215,6 +215,9 @@ def _collate(x):
         pbar.close()
         return res
 
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")
+
 
 import numpy as np
 import torchvision.transforms as T

lmms_eval/models/mplug_owl_video.py

@@ -194,3 +194,6 @@ def generate_until(self, requests) -> List[str]:
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         return super().loglikelihood(requests)
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/phi3v.py

@@ -215,3 +215,6 @@ def _collate(x):
         res = re_ords.get_original(res)
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/qwen2_vl.py

@@ -273,3 +273,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/qwen_vl.py

@@ -307,3 +307,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/qwen_vl_api.py

@@ -121,3 +121,6 @@ def flatten(self, input):
             for j in i:
                 new_list.append(j)
         return new_list
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/reka.py

@@ -193,3 +193,6 @@ def generate_until(self, requests) -> List[str]:
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         assert False, "Reka not support loglikelihood"
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/srt_api.py

@@ -293,3 +293,6 @@ async def _process(request):
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         # TODO
         assert False, "GPT4V not support"
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/tinyllava.py

@@ -408,3 +408,6 @@ def _collate(x):
 
         pbar.close()
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/video_chatgpt.py

@@ -203,3 +203,6 @@ def rank(self):
     @property
     def world_size(self):
         return self._world_size
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/video_llava.py

@@ -209,3 +209,6 @@ def generate_until(self, requests) -> List[str]:
             res.append(outputs)
             pbar.update(1)
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/vila.py

@@ -366,3 +366,6 @@ def generate_until(self, requests) -> List[str]:
             res.append(outputs)
             pbar.update(1)
         return res
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")

lmms_eval/models/xcomposer2_4KHD.py

@@ -229,6 +229,9 @@ def generate_until(self, requests) -> List[str]:
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         return super().loglikelihood(requests)
 
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")
+
 
 def padding_336(b):
     width, height = b.size

lmms_eval/models/xcomposer2d5.py

@@ -182,3 +182,6 @@ def generate_until(self, requests) -> List[str]:
 
     def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]:
         assert False, "Not implemented yet."
+
+    def generate_until_multi_round(self, requests) -> List[str]:
+        raise NotImplementedError("TODO: Implement multi-round generation")