Bring back fix for metadata

EvolvingLMMs-Lab · Sep 15, 2024 · 6442376 · 6442376
1 parent 10be8c3
commit 6442376
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py
@@ -448,7 +448,7 @@ def build_all_requests(
 
             # TODO: we should override self.config.repeats if doing greedy gen so users don't waste time+compute
             per_task_metadata = {"task": self.config["task"], "doc_id": doc_id, "repeats": self.config.repeats, "split": split}
-            if self.config.metadata:
+            if self.config.metadata and type(self.config.metadata) == dict:  # TODO: temporary fix for metadata loading, ignore the list of dict type.
                 per_task_metadata.update(self.config.metadata)
 
             inst = self.construct_requests(doc_id=doc_id, ctx=fewshot_ctx, metadata=per_task_metadata)