Fix gen kwargs image aspect ratio in internvl2

.gitignore

@@ -13,6 +13,7 @@ temp
 __pycache__
 .ipynb_checkpoints
 temp
+.DS_STORE
 # IPython
 profile_default/
 ipython_config.py

lmms_eval/__main__.py

@@ -165,7 +165,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
     # reset logger
     eval_logger.remove()
     eval_logger.add(sys.stdout, colorize=True, level=args.verbosity)
-    # eval_logger.add(sys.stderr, level=args.verbosity)
     eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 

lmms_eval/api/samplers.py

@@ -37,7 +37,9 @@ def get_context(self, doc, num_fewshot):
                     + (
                         str(self.doc_to_target(doc)[0])
                         if type(self.doc_to_target(doc)) is list
-                        else self.doc_to_target(doc) if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str) else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
+                        else self.doc_to_target(doc)
+                        if (self.config.doc_to_choice is None or type(self.doc_to_target(doc)) is str)
+                        else str(self.doc_to_choice(doc)[self.doc_to_target(doc)])
                     )
                     for doc in selected_docs
                 ]

lmms_eval/evaluator.py

@@ -325,7 +325,7 @@ def evaluate(
             # hack: remove image columns to speed avoid loading images and speed up postprocessing
             # reason: doc_iterator will actually load image if it's in the doc.
             docs = task.test_docs() if task.has_test_docs() else task.validation_docs()
-            if "d170" not in task_name and "dc100" not in task_name and "dc200" not in task_name and "llava_wilder" not in task_name and "live_bench" not in task_name and "wildvision" not in task_name:
+            if "d170" not in task_name and "dc100" not in task_name and "dc200" not in task_name and "llava_wilder" not in task_name and "livebench" not in task_name and "wildvision" not in task_name:
                 remove_cols = []
                 features = docs.features
                 # If it is an Image instance or a Sequence of Image instance. Remove it

lmms_eval/models/__init__.py

@@ -4,6 +4,10 @@
 from loguru import logger
 import sys
 
+import hf_transfer
+
+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+
 logger.remove()
 logger.add(sys.stdout, level="WARNING")
 
@@ -34,7 +38,6 @@
     "tinyllava": "TinyLlava",
     "llava_hf": "LlavaHf",
     "longva": "LongVA",
-    "llava_onevision": "Llava_OneVision",
     "llava_hf": "LlavaHf",
     "longva": "LongVA",
     "vila": "VILA",
@@ -44,8 +47,7 @@
     try:
         exec(f"from .{model_name} import {model_class}")
     except ImportError as e:
-        # logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
-        pass
+        logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
 
 if os.environ.get("LMMS_EVAL_PLUGINS", None):
     # Allow specifying other packages to import models from
@@ -55,8 +57,4 @@
             try:
                 exec(f"from {plugin}.models.{model_name} import {model_class}")
             except ImportError:
-                pass
-
-import hf_transfer
-
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+                logger.warning(f"Failed to import {model_class} from {model_name}")