From 66176a0a64651785faecdf07f9f96baf0b0a4d61 Mon Sep 17 00:00:00 2001
From: chenghao xiao <85804993+gowitheflow-1998@users.noreply.github.com>
Date: Mon, 11 Nov 2024 03:33:06 +0800
Subject: [PATCH] [mieb] mieb scripts (siglip rerun & linear probing ablation &
 params count) (#1429)

* mieb scripts

* lint
---
 scripts/run_mieb_get_params.py     | 28 ++++++++++++
 scripts/run_mieb_kshot_ablation.py | 71 ++++++++++++++++++++++++++++++
 scripts/run_mieb_rerun_siglip.py   | 30 +++++++++++++
 3 files changed, 129 insertions(+)
 create mode 100644 scripts/run_mieb_get_params.py
 create mode 100644 scripts/run_mieb_kshot_ablation.py
 create mode 100644 scripts/run_mieb_rerun_siglip.py

diff --git a/scripts/run_mieb_get_params.py b/scripts/run_mieb_get_params.py
new file mode 100644
index 000000000..a28a35ef6
--- /dev/null
+++ b/scripts/run_mieb_get_params.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import pandas as pd
+import torch
+from tqdm import tqdm
+
+import mteb
+
+params = []
+
+# add all model names
+model_names = [  # "google/siglip-base-patch16-512",
+    "google/siglip-so400m-patch14-384"
+    # ...
+]
+
+for model_name in tqdm(model_names):
+    model = mteb.get_model(model_name)
+
+    total_params = sum(p.numel() for p in model.model.parameters())
+    total_params = total_params / 1e6
+    params.append([model_name, total_params])
+
+    del model
+    torch.cuda.empty_cache()
+
+param_frame = pd.DataFrame(params, columns=["model name", "# params"])
+param_frame.to_csv("params.csv", index=False)
diff --git a/scripts/run_mieb_kshot_ablation.py b/scripts/run_mieb_kshot_ablation.py
new file mode 100644
index 000000000..a277c0d78
--- /dev/null
+++ b/scripts/run_mieb_kshot_ablation.py
@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import mteb
+
+for model_name in [
+    # key ones for this ablation (different types of models)
+    "openai/clip-vit-base-patch32",
+    "openai/clip-vit-base-patch16",
+    "openai/clip-vit-large-patch14",
+    "royokong/e5-v",
+    "facebook/dinov2-small",
+    "facebook/dinov2-base",
+    "facebook/dinov2-large",
+    "facebook/dinov2-giant",
+    # more insights
+    "BAAI/bge-visualized-base",
+    "BAAI/bge-visualized-m3",
+    "google/siglip-so400m-patch14-384",
+    "google/siglip-base-patch16-256-multilingual",
+    "google/siglip-base-patch16-256",
+    "google/siglip-base-patch16-512",
+    "google/siglip-base-patch16-384",
+    "google/siglip-base-patch16-224",
+    "google/siglip-large-patch16-256",
+    "google/siglip-large-patch16-384",
+    "nyu-visionx/moco-v3-vit-b",
+    "nyu-visionx/moco-v3-vit-l",
+    "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K",
+    "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K",
+    "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K",
+    "laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K",
+    "laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K",
+    "laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K",
+    "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k",
+    "laion/CLIP-ViT-g-14-laion2B-s34B-b88K",
+    "laion/CLIP-ViT-H-14-laion2B-s32B-b79K",
+    "laion/CLIP-ViT-L-14-laion2B-s32B-b82K",
+    "laion/CLIP-ViT-B-32-laion2B-s34B-b79K",
+    "EVA02-CLIP-B-16",
+    "EVA02-CLIP-L-14",
+    "EVA02-CLIP-bigE-14",
+    "EVA02-CLIP-bigE-14-plus",
+    "TIGER-Lab/VLM2Vec-LoRA",
+    "TIGER-Lab/VLM2Vec-Full",
+    # run if enough compute:
+    # "Salesforce/blip-itm-base-coco",
+    # "Salesforce/blip-itm-large-coco",
+    # "Salesforce/blip-itm-base-flickr",
+    # "Salesforce/blip-itm-large-flickr",
+    # "kakaobrain/align-base",
+    # "jinaai/jina-clip-v1",
+    # "nomic-ai/nomic-embed-vision-v1.5",
+    # "Salesforce/blip2-opt-2.7b",
+    # "Salesforce/blip2-opt-6.7b-coco",
+    # "embed-english-v3.0-v",  # not feasible to run due to the 40 images/min constraint
+]:
+    # 16 by default already
+
+    for k_shot in [8, 32, 64, 128, 256]:
+        model = mteb.get_model(model_name)
+        tasks = mteb.get_tasks(
+            task_types=[
+                "ImageClassification",
+            ]
+        )
+        for task in tasks:
+            task.samples_per_label = k_shot
+        evaluation = mteb.MTEB(tasks=tasks)
+        results = evaluation.run(
+            model, output_folder=f"results-mieb-final/linear_probe_{k_shot}"
+        )
diff --git a/scripts/run_mieb_rerun_siglip.py b/scripts/run_mieb_rerun_siglip.py
new file mode 100644
index 000000000..539a31e2e
--- /dev/null
+++ b/scripts/run_mieb_rerun_siglip.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import mteb
+
+for model_name in [
+    "google/siglip-so400m-patch14-384",
+    "google/siglip-base-patch16-256-multilingual",
+    "google/siglip-base-patch16-256",
+    "google/siglip-base-patch16-512",
+    "google/siglip-base-patch16-384",
+    "google/siglip-base-patch16-224",
+    "google/siglip-large-patch16-256",
+    "google/siglip-large-patch16-384",
+]:
+    model = mteb.get_model(model_name)
+    tasks = mteb.get_tasks(
+        task_types=[
+            "Any2AnyRetrieval",
+            "Any2AnyMultiChoice",
+            "Any2TextMutipleChoice",
+            "ImageClustering",
+            "ImageClassification",
+            "ImageMultilabelClassification",
+            "ImageTextPairClassification",
+            # "VisualSTS",  # visual sts does not need rerun as will be the same after fixed.
+            "ZeroShotClassification",
+        ]
+    )
+    evaluation = mteb.MTEB(tasks=tasks)
+    results = evaluation.run(model, output_folder="results-mieb-final/siglip_rerun")