Add descriptive stats to mising tasks and add number of qrels (#1476)

* add code for comupting number of qrels * add stats fever hotpotqa msmarco topiocqa * miracl mrtidy * multilongdoc miracl reranking * add multi eurlex * fix tests for descriptive stats * fix tests --------- Co-authored-by: Roman Solomatin <[email protected]>
embeddings-benchmark · Nov 21, 2024 · 0abe1a0 · 0abe1a0
1 parent 0df0210
commit 0abe1a0
Show file tree

Hide file tree

Showing 14 changed files with 6,279 additions and 9 deletions.
diff --git a/mteb/abstasks/AbsTaskRetrieval.py b/mteb/abstasks/AbsTaskRetrieval.py
@@ -58,6 +58,7 @@ class RetrievalDescriptiveStatistics(DescriptiveStatistics):
     num_samples: int
     num_queries: int
     num_documents: int
+    num_relevant_docs: int
     number_of_characters: int
 
     min_document_length: int
@@ -419,6 +420,7 @@ def _calculate_metrics_from_split(
         query_len, doc_len = calculate_length(queries, corpus)
         num_documents = len(corpus)
         num_queries = len(queries)
+        num_relevant_docs = sum(len(relevant_docs[qid]) for qid in relevant_docs)
         none_queries = sum(q is None or len(q) == 0 for q in queries.values())
 
         # create a list of number of relevant docs per query
@@ -466,6 +468,7 @@ def _calculate_metrics_from_split(
             num_samples=num_documents + num_queries,
             num_queries=num_queries,
             num_documents=num_documents,
+            num_relevant_docs=num_relevant_docs,
             min_document_length=min(doc_len),
             average_document_length=sum(doc_len) / num_documents,
             max_document_length=max(doc_len),