Skip to content

Commit

Permalink
fix: Fix test for empty descriptive tasks (#1413)
Browse files Browse the repository at this point in the history 
* fix test

* skip mock

* add message to assert

* fix test

* lint

* fix tests

* upd tests

* update descriptive stats files

* add stat to speed
  • Loading branch information
@Samoed
Samoed authored Nov 19, 2024
1 parent 0e9b6fd commit 0a5bedb
Show file tree
Hide file tree
Showing 8 changed files with 363 additions and 553 deletions.
2 changes: 1 addition & 1 deletion mteb/abstasks/AbsTaskSpeedTask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,4 +112,4 @@ def _add_main_score(self, scores) -> None:
def _calculate_metrics_from_split(
self, split: str, hf_subset: str | None = None, compute_overall: bool = False
) -> dict[str, float]:
pass
return {"num_samples": 1}
328 changes: 328 additions & 0 deletions mteb/descriptive_stats/Retrieval/IndicQARetrieval.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,328 @@
{
"test": {
"number_of_characters": 6031160,
"num_samples": 21319,
"num_queries": 18560,
"num_documents": 2759,
"min_document_length": 8,
"average_document_length": 395.7480971366437,
"max_document_length": 226,
"unique_documents": 2759,
"min_query_length": 146,
"average_query_length": 266.1255926724138,
"max_query_length": 14782,
"unique_queries": 18560,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0007543103448275,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 2759,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null,
"hf_subset_descriptive_stats": {
"as": {
"number_of_characters": 451360,
"num_samples": 2035,
"num_queries": 1785,
"num_documents": 250,
"min_document_length": 13,
"average_document_length": 404.16,
"max_document_length": 184,
"unique_documents": 250,
"min_query_length": 355,
"average_query_length": 196.2577030812325,
"max_query_length": 6654,
"unique_queries": 1785,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0016806722689076,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 250,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"bn": {
"number_of_characters": 649559,
"num_samples": 2012,
"num_queries": 1762,
"num_documents": 250,
"min_document_length": 15,
"average_document_length": 402.224,
"max_document_length": 202,
"unique_documents": 250,
"min_query_length": 684,
"average_query_length": 311.5794551645857,
"max_query_length": 6767,
"unique_queries": 1762,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0005675368898979,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 250,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"gu": {
"number_of_characters": 359851,
"num_samples": 2263,
"num_queries": 2015,
"num_documents": 248,
"min_document_length": 9,
"average_document_length": 490.51612903225805,
"max_document_length": 173,
"unique_documents": 248,
"min_query_length": 147,
"average_query_length": 118.21488833746898,
"max_query_length": 3253,
"unique_queries": 2015,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0009925558312656,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 248,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"hi": {
"number_of_characters": 747350,
"num_samples": 1805,
"num_queries": 1544,
"num_documents": 261,
"min_document_length": 19,
"average_document_length": 312.63984674329504,
"max_document_length": 138,
"unique_documents": 261,
"min_query_length": 1156,
"average_query_length": 431.18588082901556,
"max_query_length": 8857,
"unique_queries": 1544,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0019430051813472,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 261,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"kn": {
"number_of_characters": 303604,
"num_samples": 1774,
"num_queries": 1517,
"num_documents": 257,
"min_document_length": 14,
"average_document_length": 298.6031128404669,
"max_document_length": 133,
"unique_documents": 257,
"min_query_length": 146,
"average_query_length": 149.547132498352,
"max_query_length": 3130,
"unique_queries": 1517,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0,
"max_relevant_docs_per_query": 1,
"unique_relevant_docs": 257,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"ml": {
"number_of_characters": 743604,
"num_samples": 1834,
"num_queries": 1587,
"num_documents": 247,
"min_document_length": 25,
"average_document_length": 487.8987854251012,
"max_document_length": 219,
"unique_documents": 247,
"min_query_length": 859,
"average_query_length": 392.6231884057971,
"max_query_length": 11919,
"unique_queries": 1587,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0,
"max_relevant_docs_per_query": 1,
"unique_relevant_docs": 247,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"mr": {
"number_of_characters": 521991,
"num_samples": 1850,
"num_queries": 1600,
"num_documents": 250,
"min_document_length": 13,
"average_document_length": 376.224,
"max_document_length": 215,
"unique_documents": 250,
"min_query_length": 746,
"average_query_length": 267.459375,
"max_query_length": 6702,
"unique_queries": 1600,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0,
"max_relevant_docs_per_query": 1,
"unique_relevant_docs": 250,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"or": {
"number_of_characters": 294386,
"num_samples": 1928,
"num_queries": 1676,
"num_documents": 252,
"min_document_length": 8,
"average_document_length": 366.27777777777777,
"max_document_length": 195,
"unique_documents": 252,
"min_query_length": 260,
"average_query_length": 120.57517899761336,
"max_query_length": 2277,
"unique_queries": 1676,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0011933174224343,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 252,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"pa": {
"number_of_characters": 432818,
"num_samples": 1778,
"num_queries": 1537,
"num_documents": 241,
"min_document_length": 13,
"average_document_length": 372.4190871369295,
"max_document_length": 226,
"unique_documents": 241,
"min_query_length": 422,
"average_query_length": 223.20429407937542,
"max_query_length": 6082,
"unique_queries": 1537,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0013012361743656,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 241,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"ta": {
"number_of_characters": 676404,
"num_samples": 2056,
"num_queries": 1803,
"num_documents": 253,
"min_document_length": 19,
"average_document_length": 385.27272727272725,
"max_document_length": 196,
"unique_documents": 253,
"min_query_length": 769,
"average_query_length": 321.0926234054354,
"max_query_length": 6940,
"unique_queries": 1803,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0005546311702718,
"max_relevant_docs_per_query": 2,
"unique_relevant_docs": 253,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
},
"te": {
"number_of_characters": 850233,
"num_samples": 1984,
"num_queries": 1734,
"num_documents": 250,
"min_document_length": 13,
"average_document_length": 464.756,
"max_document_length": 147,
"unique_documents": 250,
"min_query_length": 1072,
"average_query_length": 423.32410611303345,
"max_query_length": 14782,
"unique_queries": 1734,
"none_queries": 0,
"min_relevant_docs_per_query": 1,
"average_relevant_docs_per_query": 1.0,
"max_relevant_docs_per_query": 1,
"unique_relevant_docs": 250,
"num_instructions": null,
"min_instruction_length": null,
"average_instruction_length": null,
"max_instruction_length": null,
"unique_instructions": null,
"min_top_ranked_per_query": null,
"average_top_ranked_per_query": null,
"max_top_ranked_per_query": null
}
}
}
}
5 changes: 5 additions & 0 deletions mteb/descriptive_stats/Speed/CPUSpeedTask.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"test": {
"num_samples": 1
}
}
5 changes: 5 additions & 0 deletions mteb/descriptive_stats/Speed/GPUSpeedTask.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"test": {
"num_samples": 1
}
}
Loading

0 comments on commit 0a5bedb

Please sign in to comment.