update 10.7 pure

.gitignore

@@ -104,3 +104,8 @@ describe/
 clip-vit-base-patch32/
 ai2d_check/
 lmms_eval/tasks/vatex/__pycache__/utils.cpython-310.pyc
+data_clean/
+gpt_4v/
+.js
+clip-vit-base-patch32/
+*.whl

lmms_eval/models/__init__.py

@@ -46,24 +46,17 @@
     "qwen":"Qwen",
     'llama':"Llama",
     "gemma":"Gemma",
-<<<<<<< HEAD
-    "cambrian":"Cambrian",
-=======
-    "cambrian_8b":"Cambrian_8b",
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
+    "cambrian_model":"Cambrian",
     "internvl2": "InternVL2",
     "mantis": "Mantis",
     "emu2": "Emu2",
     "paligemma":"Paligemma",
-<<<<<<< HEAD
     "internvl2_large":"InternVL2_large",
     "MIO_sft":"MIO",
     "onevision":"onevision",
     "onevision_large":"onevision_large",
     "cogvlm2":"cogvlm2",
     "MIO_batch":"MIO_batch"
-=======
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
 }
 
 for model_name, model_class in AVAILABLE_MODELS.items():

lmms_eval/models/cambrian.py → lmms_eval/models/cambrian_model.py

@@ -8,16 +8,6 @@
 from PIL import Image
 from datetime import timedelta
 from lmms_eval.api.registry import register_model
-# from lmms_eval.models.model_utils.qwen.qwen_generate_utils import make_context
-# 获取当前文件的目录
-# current_dir = os.path.dirname(os.path.abspath(__file__))
-# # 获取同级目录的路径
-# parent_dir = os.path.dirname(current_dir)
-
-# # 将同级目录添加到模块搜索路径
-# sys.path.append(parent_dir)
-# sys.path.append(current_dir)
-# os.chdir('./lmms_eval/models/cambrian')
 from cambrian.model.builder import load_pretrained_model
 from cambrian.conversation import conv_templates, SeparatorStyle
 from cambrian.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path
@@ -131,7 +121,7 @@ def _tokenize_str(role, content):
 
 
 
-@register_model("cambrian")
+@register_model("cambrian_model")
 class Cambrian(lmms):
     """
     cambrian_8b model
@@ -140,8 +130,7 @@ class Cambrian(lmms):
 
     def __init__(
         self,
-        pretrained: str = "/ML-A100/team/mm/zhangge/models/cambrian_8b",
-        # pretrained: str = "nyu-visionx/cambrian-8b",
+        pretrained: str = "nyu-visionx/cambrian-8b",
         device: Optional[str] = "cuda",
         device_map="auto",
         batch_size: Optional[Union[int, str]] = 1,
@@ -353,7 +342,7 @@ def _collate(x):
             for visual in visuals:
                 name = uuid.uuid4().hex.upper()[0:6]
                 visual.save(f"./lmms_eval/tmp/{name}.png")
-                visual_paths.append(f"./lmms_eval/tmp/{name}.png")
+                visual_paths.append(f"./lmms-eval/lmms_eval/tmp/{name}.png")
 
             # we assume all gen kwargs in the batch are the same
             # this is safe to assume because the `grouper` object ensures it.
@@ -370,15 +359,15 @@ def _collate(x):
                 elif not isinstance(until, list):
                     raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
             # preconfigure gen_kwargs with defaults
-            if "image_sizes" not in gen_kwargs:
-                try:
-                    gen_kwargs["image_sizes"] = [visuals[0].size]
-                except:
-                    gen_kwargs["image_sizes"] = None
+            # if "image_sizes" not in gen_kwargs:
+            #     try:
+            #         gen_kwargs["image_sizes"] = [visuals[0].size]
+            #     except:
+            #         gen_kwargs["image_sizes"] = None
             if "max_new_tokens" not in gen_kwargs:
                 gen_kwargs["max_new_tokens"] = 1024
             if "temperature" not in gen_kwargs:
-                gen_kwargs["temperature"] = 0
+                gen_kwargs["temperature"]= 0
             if "top_p" not in gen_kwargs:
                 gen_kwargs["top_p"] = None
             if "num_beams" not in gen_kwargs:
@@ -400,12 +389,13 @@ def _collate(x):
                     input_ids,
                     images=image_tensor,
                     image_sizes=image_sizes,
-                    do_sample=True if temperature > 0 else False,
-                    temperature=gen_kwargs["temperature"],
-                    num_beams=gen_kwargs["num_beams"],
-                    max_new_tokens=gen_kwargs["max_new_tokens"],
-                    # eos_token_id=eot_token_id,
-                    use_cache=True)
+                    **gen_kwargs)
+                    # do_sample=True if temperature > 0 else False,
+                    # temperature=gen_kwargs["temperature"],
+                    # num_beams=gen_kwargs["num_beams"],
+                    # max_new_tokens=gen_kwargs["max_new_tokens"],
+                    # # eos_token_id=eot_token_id,
+                    # use_cache=True)
             text_outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
 
             # cont_toks_list = cont.tolist()

lmms_eval/models/emu2.py

@@ -15,10 +15,7 @@
 from tqdm import tqdm
 from accelerate import Accelerator, DistributedType
 from accelerate.state import AcceleratorState
-<<<<<<< HEAD
 from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch
-=======
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
 
 from loguru import logger as eval_logger
 
@@ -33,10 +30,7 @@ def __init__(
         self,
         pretrained: str = "BAAI/Emu2",
         device: Optional[str] = "cuda",
-<<<<<<< HEAD
         device_map='auto',
-=======
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
         max_new_tokens: int = 256,
         batch_size: Optional[Union[int, str]] = 1,
         **kwargs,
@@ -51,7 +45,6 @@ def __init__(
         else:
             self._device = device
 
-<<<<<<< HEAD
         self._model = AutoModelForCausalLM.from_pretrained(pretrained, torch_dtype=torch.bfloat16, device_map=self._device, trust_remote_code=True)
         # with init_empty_weights():
         #     model = AutoModelForCausalLM.from_pretrained(
@@ -68,9 +61,6 @@ def __init__(
         #     model, 
         #     'local/path/to/hf/version/Emu2-Chat/model',
         #     device_map=device_map).eval()
-=======
-        self._model = AutoModelForCausalLM.from_pretrained(pretrained, torch_dtype=torch.bfloat16, device_map=self.device, trust_remote_code=True)
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
         # self._model = None
         self.model.eval()
         self.model.tie_weights()
@@ -204,15 +194,12 @@ def _collate(x):
             gen_kwargs = all_gen_kwargs[0]
             if "max_new_tokens" not in gen_kwargs:
                 gen_kwargs["max_new_tokens"] = 1024
-<<<<<<< HEAD
             if "until" in gen_kwargs:
                 until = gen_kwargs.pop("until")
                 if isinstance(until, str):
                     until = [until]
                 elif not isinstance(until, list):
                     raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
-=======
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
             contexts, all_gen_kwargs, doc_to_visuals, doc_id, tasks, splits = zip(*chunk)
             visuals = [doc_to_visual(self.task_dict[task][split][ids]) for ids, task, split, doc_to_visual in zip(doc_id, tasks, splits, doc_to_visuals)]
             # print(visuals[0])
@@ -221,24 +208,15 @@ def _collate(x):
             #     visuals = [visuals[idx][0] for idx in range(len(visuals))]  # get the first image in multi-image scenarios.
             # assert len(contexts) == self.batch_size_per_gpu, f"Expected contexts batch size {self.batch_size_per_gpu}, got {len(contexts)}"
             # assert len(visuals) == self.batch_size_per_gpu, f"Expected visuals batch size {self.batch_size_per_gpu}, got {len(visuals)}"
-<<<<<<< HEAD
-=======
-            print('')
-            print(contexts)
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
             formatted_contexts = [f"[<IMG_PLH>]{context}" for context in contexts]
             formatted_contexts[0] = formatted_contexts[0].replace('.', ':')
             # print(formatted_contexts)
             # print(visuals)
             inputs = self.model.build_input_ids(
                     text=formatted_contexts,
                     tokenizer=self.tokenizer,
-<<<<<<< HEAD
                     image=visuals,
                     # device=self.model.device,
-=======
-                    image=visuals
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
                 )
             # inputs = self.model.build_input_ids(
             #     text=formatted_contexts,
@@ -249,21 +227,12 @@ def _collate(x):
             # )
 
             outputs = self.model.generate(
-<<<<<<< HEAD
                     input_ids=inputs["input_ids"].to(self.device),
                     attention_mask=inputs["attention_mask"].to(self.device),
                     image=inputs["image"].to(self.device,torch.bfloat16),
                     # max_new_tokens=gen_kwargs["max_new_tokens"],
                     # length_penalty=-1
                     **gen_kwargs
-=======
-                    input_ids=inputs["input_ids"],
-                    attention_mask=inputs["attention_mask"],
-                    image=inputs["image"].to(torch.bfloat16),
-                    max_new_tokens=gen_kwargs["max_new_tokens"],
-                    length_penalty=-1
-                    # **gen_kwargs
->>>>>>> 865c7069caf994108f2fb1c2648cb346c8741a4e
                 )
 
             output_text = self._tokenizer.batch_decode(outputs, skip_special_tokens=True)

lmms_eval/models/gemma.py

@@ -30,7 +30,7 @@ class Gemma(lmms):
     """
     def __init__(
         self,
-        pretrained: str = "/ML-A100/team/mm/zhangge/models/gemma-1.1-7b-it", 
+        pretrained: str = "None", 
         device: Optional[str] = "cuda",
         batch_size: Optional[Union[int, str]] = 1,
         device_map="auto",