diff --git a/paths.json b/paths.json index c06885035..28fc78e66 100644 --- a/paths.json +++ b/paths.json @@ -13367,6 +13367,56 @@ "results/voyageai__voyage-multilingual-2/no_revision_available/XPQARetrieval.json", "results/voyageai__voyage-multilingual-2/no_revision_available/STSBenchmarkMultilingualSTS.json" ], + "voyageai__voyage-3": [ + "results/voyageai__voyage-3/no_revision_available/AILACasedocs.json", + "results/voyageai__voyage-3/no_revision_available/AILAStatutes.json", + "results/voyageai__voyage-3/no_revision_available/AlloprofRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/AppsRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/BSARDRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/CodeFeedbackMT.json", + "results/voyageai__voyage-3/no_revision_available/CodeFeedbackST.json", + "results/voyageai__voyage-3/no_revision_available/CodeSearchNetCCRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/CodeSearchNetRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/CodeTransOceanContest.json", + "results/voyageai__voyage-3/no_revision_available/CodeTransOceanDL.json", + "results/voyageai__voyage-3/no_revision_available/CosQA.json", + "results/voyageai__voyage-3/no_revision_available/GerDaLIRSmall.json", + "results/voyageai__voyage-3/no_revision_available/LEMBNarrativeQARetrieval.json", + "results/voyageai__voyage-3/no_revision_available/LEMBNeedleRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/LEMBPasskeyRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/LEMBQMSumRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/LEMBSummScreenFDRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/LEMBWikimQARetrieval.json", + "results/voyageai__voyage-3/no_revision_available/LeCaRDv2.json", + "results/voyageai__voyage-3/no_revision_available/LegalBenchConsumerContractsQA.json", + "results/voyageai__voyage-3/no_revision_available/LegalBenchCorporateLobbying.json", + "results/voyageai__voyage-3/no_revision_available/LegalQuAD.json", + "results/voyageai__voyage-3/no_revision_available/LegalSummarization.json", + "results/voyageai__voyage-3/no_revision_available/MIRACLRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/MintakaRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/RiaNewsRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/RuBQRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/StackOverflowQA.json", + "results/voyageai__voyage-3/no_revision_available/SyntecRetrieval.json", + "results/voyageai__voyage-3/no_revision_available/SyntheticText2SQL.json", + "results/voyageai__voyage-3/no_revision_available/XPQARetrieval.json" + ], + "voyageai__voyage-3-lite": [ + "results/voyageai__voyage-3-lite/no_revision_available/AILACasedocs.json", + "results/voyageai__voyage-3-lite/no_revision_available/AILAStatutes.json", + "results/voyageai__voyage-3-lite/no_revision_available/GerDaLIRSmall.json", + "results/voyageai__voyage-3-lite/no_revision_available/LEMBNarrativeQARetrieval.json", + "results/voyageai__voyage-3-lite/no_revision_available/LEMBNeedleRetrieval.json", + "results/voyageai__voyage-3-lite/no_revision_available/LEMBPasskeyRetrieval.json", + "results/voyageai__voyage-3-lite/no_revision_available/LEMBQMSumRetrieval.json", + "results/voyageai__voyage-3-lite/no_revision_available/LEMBSummScreenFDRetrieval.json", + "results/voyageai__voyage-3-lite/no_revision_available/LEMBWikimQARetrieval.json", + "results/voyageai__voyage-3-lite/no_revision_available/LeCaRDv2.json", + "results/voyageai__voyage-3-lite/no_revision_available/LegalBenchConsumerContractsQA.json", + "results/voyageai__voyage-3-lite/no_revision_available/LegalBenchCorporateLobbying.json", + "results/voyageai__voyage-3-lite/no_revision_available/LegalQuAD.json", + "results/voyageai__voyage-3-lite/no_revision_available/LegalSummarization.json" + ], "Muennighoff__SGPT-125M-weightedmean-msmarco-specb-bitfit": [ "results/Muennighoff__SGPT-125M-weightedmean-msmarco-specb-bitfit/no_revision_available/MassiveIntentClassification.json", "results/Muennighoff__SGPT-125M-weightedmean-msmarco-specb-bitfit/no_revision_available/BiorxivClusteringP2P.json", diff --git a/results.py b/results.py index 83272920b..2cb4b4ea4 100644 --- a/results.py +++ b/results.py @@ -497,6 +497,8 @@ "voyageai__voyage-lite-01-instruct", "voyageai__voyage-lite-02-instruct", "voyageai__voyage-multilingual-2", + "voyageai__voyage-3", + "voyageai__voyage-3-lite", "vprelovac__universal-sentence-encoder-multilingual-3", "vprelovac__universal-sentence-encoder-multilingual-large-3", ] diff --git a/results/voyageai__voyage-3-lite/no_revision_available/AILACasedocs.json b/results/voyageai__voyage-3-lite/no_revision_available/AILACasedocs.json new file mode 100644 index 000000000..978520559 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/AILACasedocs.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4106e6bcc72e0698d714ea8b101355e3e238431a", + "evaluation_time": 8.321073293685913, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.38154, + "map_at_1": 0.11774, + "map_at_10": 0.2824, + "map_at_100": 0.3326, + "map_at_1000": 0.33454, + "map_at_20": 0.30578, + "map_at_3": 0.21326, + "map_at_5": 0.24743, + "mrr_at_1": 0.36, + "mrr_at_10": 0.45733333333333326, + "mrr_at_100": 0.4695673020668209, + "mrr_at_1000": 0.4695673020668209, + "mrr_at_20": 0.4637348484848484, + "mrr_at_3": 0.4233333333333333, + "mrr_at_5": 0.44233333333333325, + "nauc_map_at_1000_diff1": 0.22680444481492215, + "nauc_map_at_1000_max": 0.058264921235812575, + "nauc_map_at_1000_std": -0.3647929169273832, + "nauc_map_at_100_diff1": 0.22312948994886786, + "nauc_map_at_100_max": 0.05478784161725515, + "nauc_map_at_100_std": -0.36448202027822085, + "nauc_map_at_10_diff1": 0.2215987871258074, + "nauc_map_at_10_max": 0.0045721676125434155, + "nauc_map_at_10_std": -0.39561512229320683, + "nauc_map_at_1_diff1": 0.33826120576351165, + "nauc_map_at_1_max": 0.06003223626014147, + "nauc_map_at_1_std": -0.3849608871379809, + "nauc_map_at_20_diff1": 0.2121001502428955, + "nauc_map_at_20_max": 0.02000717925582694, + "nauc_map_at_20_std": -0.38079097083669483, + "nauc_map_at_3_diff1": 0.3108232789388227, + "nauc_map_at_3_max": 0.07251751092473184, + "nauc_map_at_3_std": -0.3818344258445587, + "nauc_map_at_5_diff1": 0.26047649421585917, + "nauc_map_at_5_max": 0.02893656086180195, + "nauc_map_at_5_std": -0.3803407350043284, + "nauc_mrr_at_1000_diff1": 0.13590981062873708, + "nauc_mrr_at_1000_max": 0.03794546498209027, + "nauc_mrr_at_1000_std": -0.3323213254706094, + "nauc_mrr_at_100_diff1": 0.13590981062873708, + "nauc_mrr_at_100_max": 0.03794546498209027, + "nauc_mrr_at_100_std": -0.3323213254706094, + "nauc_mrr_at_10_diff1": 0.12946164961838497, + "nauc_mrr_at_10_max": 0.04253237584217536, + "nauc_mrr_at_10_std": -0.3316373685619662, + "nauc_mrr_at_1_diff1": 0.13577914239227423, + "nauc_mrr_at_1_max": -0.03183225738728817, + "nauc_mrr_at_1_std": -0.359444706870309, + "nauc_mrr_at_20_diff1": 0.1351769803558538, + "nauc_mrr_at_20_max": 0.035118723107941084, + "nauc_mrr_at_20_std": -0.3312427934370972, + "nauc_mrr_at_3_diff1": 0.1655314639498021, + "nauc_mrr_at_3_max": 0.04594602913564839, + "nauc_mrr_at_3_std": -0.3438103236756822, + "nauc_mrr_at_5_diff1": 0.14054834158345927, + "nauc_mrr_at_5_max": 0.035727431560419354, + "nauc_mrr_at_5_std": -0.34653712153173016, + "nauc_ndcg_at_1000_diff1": 0.19399576593643414, + "nauc_ndcg_at_1000_max": 0.07372089639836343, + "nauc_ndcg_at_1000_std": -0.3381474164640091, + "nauc_ndcg_at_100_diff1": 0.17564092003061438, + "nauc_ndcg_at_100_max": 0.07343870665529059, + "nauc_ndcg_at_100_std": -0.3216818651714621, + "nauc_ndcg_at_10_diff1": 0.18541150098566386, + "nauc_ndcg_at_10_max": 0.03683917481009019, + "nauc_ndcg_at_10_std": -0.3775098629847817, + "nauc_ndcg_at_1_diff1": 0.13577914239227423, + "nauc_ndcg_at_1_max": -0.03183225738728817, + "nauc_ndcg_at_1_std": -0.359444706870309, + "nauc_ndcg_at_20_diff1": 0.1489261374086848, + "nauc_ndcg_at_20_max": 0.013220882910496612, + "nauc_ndcg_at_20_std": -0.3433646168711587, + "nauc_ndcg_at_3_diff1": 0.29960048545744683, + "nauc_ndcg_at_3_max": 0.1100822360591833, + "nauc_ndcg_at_3_std": -0.36610259960246705, + "nauc_ndcg_at_5_diff1": 0.2432713686658448, + "nauc_ndcg_at_5_max": 0.10419955769718153, + "nauc_ndcg_at_5_std": -0.35193778838882284, + "nauc_precision_at_1000_diff1": 0.1569166013898087, + "nauc_precision_at_1000_max": 0.3193332683366335, + "nauc_precision_at_1000_std": 0.05502755075625906, + "nauc_precision_at_100_diff1": 0.11406300006253677, + "nauc_precision_at_100_max": 0.31741716665256575, + "nauc_precision_at_100_std": 0.07248354736776305, + "nauc_precision_at_10_diff1": 0.04641435412219487, + "nauc_precision_at_10_max": 0.053856804938556616, + "nauc_precision_at_10_std": -0.23232417931114027, + "nauc_precision_at_1_diff1": 0.13577914239227423, + "nauc_precision_at_1_max": -0.03183225738728817, + "nauc_precision_at_1_std": -0.359444706870309, + "nauc_precision_at_20_diff1": 0.05806584045364722, + "nauc_precision_at_20_max": 0.18525645960023593, + "nauc_precision_at_20_std": -0.04514920586046764, + "nauc_precision_at_3_diff1": 0.24310475917620758, + "nauc_precision_at_3_max": 0.06401031393593481, + "nauc_precision_at_3_std": -0.33590814426287297, + "nauc_precision_at_5_diff1": 0.1374756938306522, + "nauc_precision_at_5_max": 0.06846385009722461, + "nauc_precision_at_5_std": -0.2698957044369809, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.23264557740078837, + "nauc_recall_at_100_max": 0.159802171372661, + "nauc_recall_at_100_std": 0.2920852872225342, + "nauc_recall_at_10_diff1": 0.04261035679343841, + "nauc_recall_at_10_max": -0.03225531214913665, + "nauc_recall_at_10_std": -0.29384996013645287, + "nauc_recall_at_1_diff1": 0.33826120576351165, + "nauc_recall_at_1_max": 0.06003223626014147, + "nauc_recall_at_1_std": -0.3849608871379809, + "nauc_recall_at_20_diff1": -0.0412390471587858, + "nauc_recall_at_20_max": -0.07771184412953082, + "nauc_recall_at_20_std": -0.19523810164173544, + "nauc_recall_at_3_diff1": 0.29707784088826306, + "nauc_recall_at_3_max": 0.11162099294883046, + "nauc_recall_at_3_std": -0.35102019917685096, + "nauc_recall_at_5_diff1": 0.1943414621849361, + "nauc_recall_at_5_max": 0.03137833596530802, + "nauc_recall_at_5_std": -0.35136038822591403, + "ndcg_at_1": 0.36, + "ndcg_at_10": 0.38154, + "ndcg_at_100": 0.52174, + "ndcg_at_1000": 0.53267, + "ndcg_at_20": 0.4218, + "ndcg_at_3": 0.34325, + "ndcg_at_5": 0.34495, + "precision_at_1": 0.36, + "precision_at_10": 0.164, + "precision_at_100": 0.0362, + "precision_at_1000": 0.0039, + "precision_at_20": 0.111, + "precision_at_3": 0.28667, + "precision_at_5": 0.232, + "recall_at_1": 0.11774, + "recall_at_10": 0.44887, + "recall_at_100": 0.95603, + "recall_at_1000": 1.0, + "recall_at_20": 0.56508, + "recall_at_3": 0.24251, + "recall_at_5": 0.31412 + } + ] + }, + "task_name": "AILACasedocs" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/AILAStatutes.json b/results/voyageai__voyage-3-lite/no_revision_available/AILAStatutes.json new file mode 100644 index 000000000..2c02bd76e --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/AILAStatutes.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e", + "evaluation_time": 3.139857292175293, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.35033, + "map_at_1": 0.08, + "map_at_10": 0.21573, + "map_at_100": 0.293, + "map_at_1000": 0.293, + "map_at_20": 0.25577, + "map_at_3": 0.14072, + "map_at_5": 0.17969, + "mrr_at_1": 0.34, + "mrr_at_10": 0.5111031746031744, + "mrr_at_100": 0.5151237373737372, + "mrr_at_1000": 0.5151237373737372, + "mrr_at_20": 0.5141713564213563, + "mrr_at_3": 0.43999999999999995, + "mrr_at_5": 0.4919999999999998, + "nauc_map_at_1000_diff1": 0.2627428049451818, + "nauc_map_at_1000_max": 0.37195611611594037, + "nauc_map_at_1000_std": 0.1695908199424899, + "nauc_map_at_100_diff1": 0.2627428049451818, + "nauc_map_at_100_max": 0.37195611611594037, + "nauc_map_at_100_std": 0.1695908199424899, + "nauc_map_at_10_diff1": 0.257026135451343, + "nauc_map_at_10_max": 0.358611488616898, + "nauc_map_at_10_std": 0.17561884128144586, + "nauc_map_at_1_diff1": 0.278328846552081, + "nauc_map_at_1_max": 0.43277593704700756, + "nauc_map_at_1_std": 0.30501138952164003, + "nauc_map_at_20_diff1": 0.2415186441540987, + "nauc_map_at_20_max": 0.3631129644969627, + "nauc_map_at_20_std": 0.180440504561769, + "nauc_map_at_3_diff1": 0.2914087617281748, + "nauc_map_at_3_max": 0.37794665974262803, + "nauc_map_at_3_std": 0.16516069465199582, + "nauc_map_at_5_diff1": 0.2346491426208135, + "nauc_map_at_5_max": 0.3591654170954656, + "nauc_map_at_5_std": 0.1769291260967869, + "nauc_mrr_at_1000_diff1": 0.3296532710194783, + "nauc_mrr_at_1000_max": 0.4148119538338282, + "nauc_mrr_at_1000_std": 0.29561611808491245, + "nauc_mrr_at_100_diff1": 0.3296532710194783, + "nauc_mrr_at_100_max": 0.4148119538338282, + "nauc_mrr_at_100_std": 0.29561611808491245, + "nauc_mrr_at_10_diff1": 0.3226708021374957, + "nauc_mrr_at_10_max": 0.4105946066597358, + "nauc_mrr_at_10_std": 0.2948483005668789, + "nauc_mrr_at_1_diff1": 0.3545138980895093, + "nauc_mrr_at_1_max": 0.45496701025464353, + "nauc_mrr_at_1_std": 0.3394101592517023, + "nauc_mrr_at_20_diff1": 0.32897595485222514, + "nauc_mrr_at_20_max": 0.4128928483014366, + "nauc_mrr_at_20_std": 0.29408727186886474, + "nauc_mrr_at_3_diff1": 0.36847244552234065, + "nauc_mrr_at_3_max": 0.41595134933827443, + "nauc_mrr_at_3_std": 0.23793952275844535, + "nauc_mrr_at_5_diff1": 0.2976288373079473, + "nauc_mrr_at_5_max": 0.4014494884923278, + "nauc_mrr_at_5_std": 0.278804694706051, + "nauc_ndcg_at_1000_diff1": 0.2875011003018454, + "nauc_ndcg_at_1000_max": 0.3778453138654806, + "nauc_ndcg_at_1000_std": 0.195237859804941, + "nauc_ndcg_at_100_diff1": 0.2875011003018454, + "nauc_ndcg_at_100_max": 0.3778453138654806, + "nauc_ndcg_at_100_std": 0.195237859804941, + "nauc_ndcg_at_10_diff1": 0.2616213255109694, + "nauc_ndcg_at_10_max": 0.351841818006536, + "nauc_ndcg_at_10_std": 0.20553076558837527, + "nauc_ndcg_at_1_diff1": 0.3545138980895093, + "nauc_ndcg_at_1_max": 0.45496701025464353, + "nauc_ndcg_at_1_std": 0.3394101592517023, + "nauc_ndcg_at_20_diff1": 0.23366274364875173, + "nauc_ndcg_at_20_max": 0.3366167411090695, + "nauc_ndcg_at_20_std": 0.17776346311216973, + "nauc_ndcg_at_3_diff1": 0.3281798973074511, + "nauc_ndcg_at_3_max": 0.3641306132599758, + "nauc_ndcg_at_3_std": 0.1546357549276064, + "nauc_ndcg_at_5_diff1": 0.21731337842351942, + "nauc_ndcg_at_5_max": 0.36072544427054365, + "nauc_ndcg_at_5_std": 0.19313711550233276, + "nauc_precision_at_1000_diff1": 0.051517535354273766, + "nauc_precision_at_1000_max": -0.086569949324014, + "nauc_precision_at_1000_std": -0.1009311940871905, + "nauc_precision_at_100_diff1": 0.05151753535427775, + "nauc_precision_at_100_max": -0.08656994932401264, + "nauc_precision_at_100_std": -0.10093119408718829, + "nauc_precision_at_10_diff1": 0.18854668275574313, + "nauc_precision_at_10_max": 0.21657099680849856, + "nauc_precision_at_10_std": 0.12317165315259393, + "nauc_precision_at_1_diff1": 0.3545138980895093, + "nauc_precision_at_1_max": 0.45496701025464353, + "nauc_precision_at_1_std": 0.3394101592517023, + "nauc_precision_at_20_diff1": 0.08028870363117518, + "nauc_precision_at_20_max": 0.12511423189540946, + "nauc_precision_at_20_std": 0.04167366045618141, + "nauc_precision_at_3_diff1": 0.324571784490813, + "nauc_precision_at_3_max": 0.31826533790096534, + "nauc_precision_at_3_std": 0.043864839613827686, + "nauc_precision_at_5_diff1": 0.11475019825535299, + "nauc_precision_at_5_max": 0.26426116838488006, + "nauc_precision_at_5_std": 0.12357916997092283, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.17516056043860467, + "nauc_recall_at_10_max": 0.23446871464624489, + "nauc_recall_at_10_std": 0.1445740144460882, + "nauc_recall_at_1_diff1": 0.278328846552081, + "nauc_recall_at_1_max": 0.43277593704700756, + "nauc_recall_at_1_std": 0.30501138952164003, + "nauc_recall_at_20_diff1": 0.07020015280087248, + "nauc_recall_at_20_max": 0.16190028721661406, + "nauc_recall_at_20_std": 0.05956946107050535, + "nauc_recall_at_3_diff1": 0.2737050070501919, + "nauc_recall_at_3_max": 0.32529362331371153, + "nauc_recall_at_3_std": 0.05578741954903067, + "nauc_recall_at_5_diff1": 0.08950675622720845, + "nauc_recall_at_5_max": 0.26082033645135566, + "nauc_recall_at_5_std": 0.11142911147890128, + "ndcg_at_1": 0.34, + "ndcg_at_10": 0.35033, + "ndcg_at_100": 0.55089, + "ndcg_at_1000": 0.55089, + "ndcg_at_20": 0.43137, + "ndcg_at_3": 0.27508, + "ndcg_at_5": 0.28528, + "precision_at_1": 0.34, + "precision_at_10": 0.176, + "precision_at_100": 0.0434, + "precision_at_1000": 0.00434, + "precision_at_20": 0.133, + "precision_at_3": 0.25333, + "precision_at_5": 0.24, + "recall_at_1": 0.08, + "recall_at_10": 0.40733, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.59933, + "recall_at_3": 0.18267, + "recall_at_5": 0.28633 + } + ] + }, + "task_name": "AILAStatutes" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/GerDaLIRSmall.json b/results/voyageai__voyage-3-lite/no_revision_available/GerDaLIRSmall.json new file mode 100644 index 000000000..645c4601b --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/GerDaLIRSmall.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "48327de6ee192e9610f3069789719788957c7abd", + "evaluation_time": 365.89770317077637, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "deu-Latn" + ], + "main_score": 0.43726, + "map_at_1": 0.28894, + "map_at_10": 0.38732, + "map_at_100": 0.39601, + "map_at_1000": 0.39658, + "map_at_20": 0.39235, + "map_at_3": 0.359, + "map_at_5": 0.37604, + "mrr_at_1": 0.3189471963380742, + "mrr_at_10": 0.4055104691323416, + "mrr_at_100": 0.4129414142837904, + "mrr_at_1000": 0.41343213284486824, + "mrr_at_20": 0.40981292983741746, + "mrr_at_3": 0.3811781374312039, + "mrr_at_5": 0.3959811454416629, + "nauc_map_at_1000_diff1": 0.4218725234665426, + "nauc_map_at_1000_max": 0.3224163971384897, + "nauc_map_at_1000_std": 0.036137781327541446, + "nauc_map_at_100_diff1": 0.42171063841689393, + "nauc_map_at_100_max": 0.32249692819449666, + "nauc_map_at_100_std": 0.03637433939031627, + "nauc_map_at_10_diff1": 0.42379325793916744, + "nauc_map_at_10_max": 0.32093958171175596, + "nauc_map_at_10_std": 0.030414384554955733, + "nauc_map_at_1_diff1": 0.48746475425934815, + "nauc_map_at_1_max": 0.2888650049308207, + "nauc_map_at_1_std": -0.007860371158818028, + "nauc_map_at_20_diff1": 0.4221365740037529, + "nauc_map_at_20_max": 0.32226700925753216, + "nauc_map_at_20_std": 0.03431537833879477, + "nauc_map_at_3_diff1": 0.4369422578391192, + "nauc_map_at_3_max": 0.3137270401649117, + "nauc_map_at_3_std": 0.01218853800951113, + "nauc_map_at_5_diff1": 0.4277880979177284, + "nauc_map_at_5_max": 0.31835118828011316, + "nauc_map_at_5_std": 0.02249946049750433, + "nauc_mrr_at_1000_diff1": 0.42501957462957973, + "nauc_mrr_at_1000_max": 0.3325829833795039, + "nauc_mrr_at_1000_std": 0.04622837553779156, + "nauc_mrr_at_100_diff1": 0.42485343277304105, + "nauc_mrr_at_100_max": 0.3326569916857509, + "nauc_mrr_at_100_std": 0.04646771815519155, + "nauc_mrr_at_10_diff1": 0.42619298289176205, + "nauc_mrr_at_10_max": 0.33224668917256833, + "nauc_mrr_at_10_std": 0.042536914544972604, + "nauc_mrr_at_1_diff1": 0.48460808764258884, + "nauc_mrr_at_1_max": 0.31629024022115826, + "nauc_mrr_at_1_std": 0.008448340417487854, + "nauc_mrr_at_20_diff1": 0.42502456535883815, + "nauc_mrr_at_20_max": 0.3328688172116466, + "nauc_mrr_at_20_std": 0.04518418867695188, + "nauc_mrr_at_3_diff1": 0.437054790494308, + "nauc_mrr_at_3_max": 0.3288489034666709, + "nauc_mrr_at_3_std": 0.02681344557608346, + "nauc_mrr_at_5_diff1": 0.4296557228104733, + "nauc_mrr_at_5_max": 0.330770583769514, + "nauc_mrr_at_5_std": 0.036199210065614745, + "nauc_ndcg_at_1000_diff1": 0.3954165954450355, + "nauc_ndcg_at_1000_max": 0.3346978064288283, + "nauc_ndcg_at_1000_std": 0.07399821733952902, + "nauc_ndcg_at_100_diff1": 0.38959549033205637, + "nauc_ndcg_at_100_max": 0.3360649078655632, + "nauc_ndcg_at_100_std": 0.08241785354589433, + "nauc_ndcg_at_10_diff1": 0.39832820199152497, + "nauc_ndcg_at_10_max": 0.3322771925164499, + "nauc_ndcg_at_10_std": 0.056745040225612844, + "nauc_ndcg_at_1_diff1": 0.48460808764258884, + "nauc_ndcg_at_1_max": 0.31629024022115826, + "nauc_ndcg_at_1_std": 0.008448340417487854, + "nauc_ndcg_at_20_diff1": 0.3925164671620135, + "nauc_ndcg_at_20_max": 0.33627290156622364, + "nauc_ndcg_at_20_std": 0.06979788040729733, + "nauc_ndcg_at_3_diff1": 0.4225531988188681, + "nauc_ndcg_at_3_max": 0.32184419432661515, + "nauc_ndcg_at_3_std": 0.021644521299644604, + "nauc_ndcg_at_5_diff1": 0.4077312319856797, + "nauc_ndcg_at_5_max": 0.32746064145437015, + "nauc_ndcg_at_5_std": 0.038997228205871745, + "nauc_precision_at_1000_diff1": 0.014653518926141607, + "nauc_precision_at_1000_max": 0.22277939303339409, + "nauc_precision_at_1000_std": 0.200875670592928, + "nauc_precision_at_100_diff1": 0.0792290190694836, + "nauc_precision_at_100_max": 0.29012044941813925, + "nauc_precision_at_100_std": 0.25545650342604076, + "nauc_precision_at_10_diff1": 0.22145123535797615, + "nauc_precision_at_10_max": 0.34007011208538995, + "nauc_precision_at_10_std": 0.14622201736832685, + "nauc_precision_at_1_diff1": 0.48460808764258884, + "nauc_precision_at_1_max": 0.31629024022115826, + "nauc_precision_at_1_std": 0.008448340417487854, + "nauc_precision_at_20_diff1": 0.16921687771883281, + "nauc_precision_at_20_max": 0.3369226442247652, + "nauc_precision_at_20_std": 0.19609287999954655, + "nauc_precision_at_3_diff1": 0.33251430153517864, + "nauc_precision_at_3_max": 0.34083297391761125, + "nauc_precision_at_3_std": 0.05290455174821755, + "nauc_precision_at_5_diff1": 0.27786822290094637, + "nauc_precision_at_5_max": 0.342957155320767, + "nauc_precision_at_5_std": 0.09330741770964344, + "nauc_recall_at_1000_diff1": 0.20045814162137127, + "nauc_recall_at_1000_max": 0.37896778961304517, + "nauc_recall_at_1000_std": 0.34566756846277447, + "nauc_recall_at_100_diff1": 0.2134373592965021, + "nauc_recall_at_100_max": 0.36762753804035536, + "nauc_recall_at_100_std": 0.32156314244165174, + "nauc_recall_at_10_diff1": 0.3081919063942128, + "nauc_recall_at_10_max": 0.3473724682762936, + "nauc_recall_at_10_std": 0.1289377251420428, + "nauc_recall_at_1_diff1": 0.48746475425934815, + "nauc_recall_at_1_max": 0.2888650049308207, + "nauc_recall_at_1_std": -0.007860371158818028, + "nauc_recall_at_20_diff1": 0.27328027197285926, + "nauc_recall_at_20_max": 0.362326990305771, + "nauc_recall_at_20_std": 0.1893054635040052, + "nauc_recall_at_3_diff1": 0.37715831679189016, + "nauc_recall_at_3_max": 0.32214886466553827, + "nauc_recall_at_3_std": 0.032772733961956894, + "nauc_recall_at_5_diff1": 0.34118446849828077, + "nauc_recall_at_5_max": 0.33454670797939395, + "nauc_recall_at_5_std": 0.07357679385017991, + "ndcg_at_1": 0.31895, + "ndcg_at_10": 0.43726, + "ndcg_at_100": 0.4784, + "ndcg_at_1000": 0.49391, + "ndcg_at_20": 0.45415, + "ndcg_at_3": 0.38398, + "ndcg_at_5": 0.41221, + "precision_at_1": 0.31895, + "precision_at_10": 0.06652, + "precision_at_100": 0.00894, + "precision_at_1000": 0.00103, + "precision_at_20": 0.03717, + "precision_at_3": 0.16473, + "precision_at_5": 0.11497, + "recall_at_1": 0.28894, + "recall_at_10": 0.57327, + "recall_at_100": 0.76163, + "recall_at_1000": 0.88289, + "recall_at_20": 0.6369, + "recall_at_3": 0.43248, + "recall_at_5": 0.4995 + } + ] + }, + "task_name": "GerDaLIRSmall" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LEMBNarrativeQARetrieval.json b/results/voyageai__voyage-3-lite/no_revision_available/LEMBNarrativeQARetrieval.json new file mode 100644 index 000000000..56f8e80c7 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LEMBNarrativeQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 126.63822603225708, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.51671, + "map_at_1": 0.40415, + "map_at_10": 0.47727, + "map_at_100": 0.4859, + "map_at_1000": 0.48661, + "map_at_20": 0.4819, + "map_at_3": 0.45612, + "map_at_5": 0.46815, + "mrr_at_1": 0.40415350751268064, + "mrr_at_10": 0.4772710079342286, + "mrr_at_100": 0.4858996450378015, + "mrr_at_1000": 0.4866144970424086, + "mrr_at_20": 0.48190001575360925, + "mrr_at_3": 0.456120202890229, + "mrr_at_5": 0.46814527706000336, + "nauc_map_at_1000_diff1": 0.6415546900551692, + "nauc_map_at_1000_max": 0.38533319986209363, + "nauc_map_at_1000_std": 0.057853754751372055, + "nauc_map_at_100_diff1": 0.6411567657158878, + "nauc_map_at_100_max": 0.3850070662513045, + "nauc_map_at_100_std": 0.058072714714432275, + "nauc_map_at_10_diff1": 0.6407517643628899, + "nauc_map_at_10_max": 0.386342846903275, + "nauc_map_at_10_std": 0.05713903641819142, + "nauc_map_at_1_diff1": 0.6952537356947764, + "nauc_map_at_1_max": 0.4084055798036384, + "nauc_map_at_1_std": 0.04305516098559638, + "nauc_map_at_20_diff1": 0.6404911403397369, + "nauc_map_at_20_max": 0.3850611967020663, + "nauc_map_at_20_std": 0.05799838248237846, + "nauc_map_at_3_diff1": 0.6501702690815119, + "nauc_map_at_3_max": 0.39374810528816845, + "nauc_map_at_3_std": 0.053974344004520766, + "nauc_map_at_5_diff1": 0.6434647686554883, + "nauc_map_at_5_max": 0.389234090800636, + "nauc_map_at_5_std": 0.05495679761087531, + "nauc_mrr_at_1000_diff1": 0.6415546900551692, + "nauc_mrr_at_1000_max": 0.38533319986209363, + "nauc_mrr_at_1000_std": 0.057853754751372055, + "nauc_mrr_at_100_diff1": 0.6411567657158878, + "nauc_mrr_at_100_max": 0.3850070662513045, + "nauc_mrr_at_100_std": 0.058072714714432275, + "nauc_mrr_at_10_diff1": 0.6407517643628899, + "nauc_mrr_at_10_max": 0.386342846903275, + "nauc_mrr_at_10_std": 0.05713903641819142, + "nauc_mrr_at_1_diff1": 0.6952537356947764, + "nauc_mrr_at_1_max": 0.4084055798036384, + "nauc_mrr_at_1_std": 0.04305516098559638, + "nauc_mrr_at_20_diff1": 0.6404911403397369, + "nauc_mrr_at_20_max": 0.3850611967020663, + "nauc_mrr_at_20_std": 0.05799838248237846, + "nauc_mrr_at_3_diff1": 0.6501702690815119, + "nauc_mrr_at_3_max": 0.39374810528816845, + "nauc_mrr_at_3_std": 0.053974344004520766, + "nauc_mrr_at_5_diff1": 0.6434647686554883, + "nauc_mrr_at_5_max": 0.389234090800636, + "nauc_mrr_at_5_std": 0.05495679761087531, + "nauc_ndcg_at_1000_diff1": 0.6283393949573638, + "nauc_ndcg_at_1000_max": 0.37608804748811764, + "nauc_ndcg_at_1000_std": 0.06330820286939626, + "nauc_ndcg_at_100_diff1": 0.6184901455989237, + "nauc_ndcg_at_100_max": 0.3668738682682804, + "nauc_ndcg_at_100_std": 0.06916103564278617, + "nauc_ndcg_at_10_diff1": 0.6158131188259138, + "nauc_ndcg_at_10_max": 0.3724820726487905, + "nauc_ndcg_at_10_std": 0.06529950222080008, + "nauc_ndcg_at_1_diff1": 0.6952537356947764, + "nauc_ndcg_at_1_max": 0.4084055798036384, + "nauc_ndcg_at_1_std": 0.04305516098559638, + "nauc_ndcg_at_20_diff1": 0.6142872726240235, + "nauc_ndcg_at_20_max": 0.36719522810653044, + "nauc_ndcg_at_20_std": 0.06871855062072717, + "nauc_ndcg_at_3_diff1": 0.6353924277902078, + "nauc_ndcg_at_3_max": 0.3881820342833853, + "nauc_ndcg_at_3_std": 0.05733853952349176, + "nauc_ndcg_at_5_diff1": 0.622754637731489, + "nauc_ndcg_at_5_max": 0.379782818936707, + "nauc_ndcg_at_5_std": 0.05943147050306446, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.44470187958147167, + "nauc_precision_at_100_max": 0.19938894293531795, + "nauc_precision_at_100_std": 0.1737110741670084, + "nauc_precision_at_10_diff1": 0.5254837483829923, + "nauc_precision_at_10_max": 0.32010011850428804, + "nauc_precision_at_10_std": 0.09675992706700258, + "nauc_precision_at_1_diff1": 0.6952537356947764, + "nauc_precision_at_1_max": 0.4084055798036384, + "nauc_precision_at_1_std": 0.04305516098559638, + "nauc_precision_at_20_diff1": 0.5047643916475815, + "nauc_precision_at_20_max": 0.28571835855904226, + "nauc_precision_at_20_std": 0.11839332297950537, + "nauc_precision_at_3_diff1": 0.5913172176970312, + "nauc_precision_at_3_max": 0.37133654004496175, + "nauc_precision_at_3_std": 0.06730327385845859, + "nauc_precision_at_5_diff1": 0.5562066400888251, + "nauc_precision_at_5_max": 0.34875173191540026, + "nauc_precision_at_5_std": 0.07379626960566459, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4447018795814721, + "nauc_recall_at_100_max": 0.1993889429353176, + "nauc_recall_at_100_std": 0.17371107416700923, + "nauc_recall_at_10_diff1": 0.5254837483829923, + "nauc_recall_at_10_max": 0.32010011850428793, + "nauc_recall_at_10_std": 0.0967599270670033, + "nauc_recall_at_1_diff1": 0.6952537356947764, + "nauc_recall_at_1_max": 0.4084055798036384, + "nauc_recall_at_1_std": 0.04305516098559638, + "nauc_recall_at_20_diff1": 0.504764391647582, + "nauc_recall_at_20_max": 0.285718358559042, + "nauc_recall_at_20_std": 0.11839332297950506, + "nauc_recall_at_3_diff1": 0.5913172176970306, + "nauc_recall_at_3_max": 0.3713365400449616, + "nauc_recall_at_3_std": 0.0673032738584581, + "nauc_recall_at_5_diff1": 0.5562066400888255, + "nauc_recall_at_5_max": 0.34875173191539993, + "nauc_recall_at_5_std": 0.07379626960566461, + "ndcg_at_1": 0.40415, + "ndcg_at_10": 0.51671, + "ndcg_at_100": 0.56396, + "ndcg_at_1000": 0.58048, + "ndcg_at_20": 0.53348, + "ndcg_at_3": 0.47296, + "ndcg_at_5": 0.49468, + "precision_at_1": 0.40415, + "precision_at_10": 0.06425, + "precision_at_100": 0.00876, + "precision_at_1000": 0.001, + "precision_at_20": 0.03543, + "precision_at_3": 0.17389, + "precision_at_5": 0.1149, + "recall_at_1": 0.40415, + "recall_at_10": 0.64245, + "recall_at_100": 0.87606, + "recall_at_1000": 1.0, + "recall_at_20": 0.70858, + "recall_at_3": 0.52168, + "recall_at_5": 0.5745 + } + ] + }, + "task_name": "LEMBNarrativeQARetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LEMBNeedleRetrieval.json b/results/voyageai__voyage-3-lite/no_revision_available/LEMBNeedleRetrieval.json new file mode 100644 index 000000000..fc6e36b7e --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LEMBNeedleRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 41.99341297149658, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test_1024": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.78, + "map_at_1": 0.78, + "map_at_10": 0.86841, + "map_at_100": 0.86841, + "map_at_1000": 0.86841, + "map_at_20": 0.86841, + "map_at_3": 0.86333, + "map_at_5": 0.86333, + "mrr_at_1": 0.78, + "mrr_at_10": 0.8684126984126986, + "mrr_at_100": 0.8684126984126986, + "mrr_at_1000": 0.8684126984126986, + "mrr_at_20": 0.8684126984126986, + "mrr_at_3": 0.8633333333333334, + "mrr_at_5": 0.8633333333333334, + "nauc_map_at_1000_diff1": 0.6484627883199919, + "nauc_map_at_1000_max": 0.2489944140511131, + "nauc_map_at_1000_std": -0.45575149103078433, + "nauc_map_at_100_diff1": 0.6484627883199919, + "nauc_map_at_100_max": 0.2489944140511131, + "nauc_map_at_100_std": -0.45575149103078433, + "nauc_map_at_10_diff1": 0.6484627883199919, + "nauc_map_at_10_max": 0.2489944140511131, + "nauc_map_at_10_std": -0.45575149103078433, + "nauc_map_at_1_diff1": 0.6592942922056846, + "nauc_map_at_1_max": 0.16794176287847168, + "nauc_map_at_1_std": -0.34269949459822924, + "nauc_map_at_20_diff1": 0.6484627883199919, + "nauc_map_at_20_max": 0.2489944140511131, + "nauc_map_at_20_std": -0.45575149103078433, + "nauc_map_at_3_diff1": 0.6547038625560562, + "nauc_map_at_3_max": 0.26666422278431534, + "nauc_map_at_3_std": -0.4864425626550342, + "nauc_map_at_5_diff1": 0.6547038625560562, + "nauc_map_at_5_max": 0.26666422278431534, + "nauc_map_at_5_std": -0.4864425626550342, + "nauc_mrr_at_1000_diff1": 0.6484627883199919, + "nauc_mrr_at_1000_max": 0.2489944140511131, + "nauc_mrr_at_1000_std": -0.45575149103078433, + "nauc_mrr_at_100_diff1": 0.6484627883199919, + "nauc_mrr_at_100_max": 0.2489944140511131, + "nauc_mrr_at_100_std": -0.45575149103078433, + "nauc_mrr_at_10_diff1": 0.6484627883199919, + "nauc_mrr_at_10_max": 0.2489944140511131, + "nauc_mrr_at_10_std": -0.45575149103078433, + "nauc_mrr_at_1_diff1": 0.6592942922056846, + "nauc_mrr_at_1_max": 0.16794176287847168, + "nauc_mrr_at_1_std": -0.34269949459822924, + "nauc_mrr_at_20_diff1": 0.6484627883199919, + "nauc_mrr_at_20_max": 0.2489944140511131, + "nauc_mrr_at_20_std": -0.45575149103078433, + "nauc_mrr_at_3_diff1": 0.6547038625560562, + "nauc_mrr_at_3_max": 0.26666422278431534, + "nauc_mrr_at_3_std": -0.4864425626550342, + "nauc_mrr_at_5_diff1": 0.6547038625560562, + "nauc_mrr_at_5_max": 0.26666422278431534, + "nauc_mrr_at_5_std": -0.4864425626550342, + "nauc_ndcg_at_1000_diff1": 0.6487016456486254, + "nauc_ndcg_at_1000_max": 0.25761207411847803, + "nauc_ndcg_at_1000_std": -0.46693424942547873, + "nauc_ndcg_at_100_diff1": 0.6487016456486254, + "nauc_ndcg_at_100_max": 0.25761207411847803, + "nauc_ndcg_at_100_std": -0.46693424942547873, + "nauc_ndcg_at_10_diff1": 0.6487016456486254, + "nauc_ndcg_at_10_max": 0.25761207411847803, + "nauc_ndcg_at_10_std": -0.46693424942547873, + "nauc_ndcg_at_1_diff1": 0.6592942922056846, + "nauc_ndcg_at_1_max": 0.16794176287847168, + "nauc_ndcg_at_1_std": -0.34269949459822924, + "nauc_ndcg_at_20_diff1": 0.6487016456486254, + "nauc_ndcg_at_20_max": 0.25761207411847803, + "nauc_ndcg_at_20_std": -0.46693424942547873, + "nauc_ndcg_at_3_diff1": 0.6656181162075767, + "nauc_ndcg_at_3_max": 0.3116181540894332, + "nauc_ndcg_at_3_std": -0.5534669301697986, + "nauc_ndcg_at_5_diff1": 0.6656181162075767, + "nauc_ndcg_at_5_max": 0.3116181540894332, + "nauc_ndcg_at_5_std": -0.5534669301697986, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.6592942922056846, + "nauc_precision_at_1_max": 0.16794176287847168, + "nauc_precision_at_1_std": -0.34269949459822924, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.7770774976657359, + "nauc_precision_at_3_max": 0.7117180205415506, + "nauc_precision_at_3_std": -1.151727357609717, + "nauc_precision_at_5_diff1": 0.7770774976657333, + "nauc_precision_at_5_max": 0.7117180205415553, + "nauc_precision_at_5_std": -1.1517273576096958, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.6592942922056846, + "nauc_recall_at_1_max": 0.16794176287847168, + "nauc_recall_at_1_std": -0.34269949459822924, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.7770774976657333, + "nauc_recall_at_3_max": 0.71171802054155, + "nauc_recall_at_3_std": -1.1517273576097067, + "nauc_recall_at_5_diff1": 0.7770774976657333, + "nauc_recall_at_5_max": 0.71171802054155, + "nauc_recall_at_5_std": -1.1517273576097067, + "ndcg_at_1": 0.78, + "ndcg_at_10": 0.90102, + "ndcg_at_100": 0.90102, + "ndcg_at_1000": 0.90102, + "ndcg_at_20": 0.90102, + "ndcg_at_3": 0.88833, + "ndcg_at_5": 0.88833, + "precision_at_1": 0.78, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.32, + "precision_at_5": 0.192, + "recall_at_1": 0.78, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.96, + "recall_at_5": 0.96 + } + ], + "test_16384": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.28, + "map_at_1": 0.28, + "map_at_10": 0.45956, + "map_at_100": 0.47292, + "map_at_1000": 0.47292, + "map_at_20": 0.47218, + "map_at_3": 0.41667, + "map_at_5": 0.44867, + "mrr_at_1": 0.28, + "mrr_at_10": 0.4595555555555555, + "mrr_at_100": 0.4729209020826668, + "mrr_at_1000": 0.4729209020826668, + "mrr_at_20": 0.47218016134192603, + "mrr_at_3": 0.41666666666666663, + "mrr_at_5": 0.4486666666666666, + "nauc_map_at_1000_diff1": 0.1142670182356824, + "nauc_map_at_1000_max": -0.14325709875531234, + "nauc_map_at_1000_std": -0.07038073417970749, + "nauc_map_at_100_diff1": 0.1142670182356824, + "nauc_map_at_100_max": -0.14325709875531234, + "nauc_map_at_100_std": -0.07038073417970749, + "nauc_map_at_10_diff1": 0.10231192802086397, + "nauc_map_at_10_max": -0.1313178597258763, + "nauc_map_at_10_std": -0.07014184007435192, + "nauc_map_at_1_diff1": 0.15383540979942995, + "nauc_map_at_1_max": -0.09134588524498606, + "nauc_map_at_1_std": -0.06288060785518773, + "nauc_map_at_20_diff1": 0.11538191057015515, + "nauc_map_at_20_max": -0.14581305804422995, + "nauc_map_at_20_std": -0.06837188873181164, + "nauc_map_at_3_diff1": 0.11314460237284153, + "nauc_map_at_3_max": -0.14140256078938107, + "nauc_map_at_3_std": -0.1014589451427227, + "nauc_map_at_5_diff1": 0.12355596018862651, + "nauc_map_at_5_max": -0.14684607246100784, + "nauc_map_at_5_std": -0.05334309239162413, + "nauc_mrr_at_1000_diff1": 0.1142670182356824, + "nauc_mrr_at_1000_max": -0.14325709875531234, + "nauc_mrr_at_1000_std": -0.07038073417970749, + "nauc_mrr_at_100_diff1": 0.1142670182356824, + "nauc_mrr_at_100_max": -0.14325709875531234, + "nauc_mrr_at_100_std": -0.07038073417970749, + "nauc_mrr_at_10_diff1": 0.10231192802086397, + "nauc_mrr_at_10_max": -0.1313178597258763, + "nauc_mrr_at_10_std": -0.07014184007435192, + "nauc_mrr_at_1_diff1": 0.15383540979942995, + "nauc_mrr_at_1_max": -0.09134588524498606, + "nauc_mrr_at_1_std": -0.06288060785518773, + "nauc_mrr_at_20_diff1": 0.11538191057015515, + "nauc_mrr_at_20_max": -0.14581305804422995, + "nauc_mrr_at_20_std": -0.06837188873181164, + "nauc_mrr_at_3_diff1": 0.11314460237284153, + "nauc_mrr_at_3_max": -0.14140256078938107, + "nauc_mrr_at_3_std": -0.1014589451427227, + "nauc_mrr_at_5_diff1": 0.12355596018862651, + "nauc_mrr_at_5_max": -0.14684607246100784, + "nauc_mrr_at_5_std": -0.05334309239162413, + "nauc_ndcg_at_1000_diff1": 0.11535802471187034, + "nauc_ndcg_at_1000_max": -0.14740997248620716, + "nauc_ndcg_at_1000_std": -0.06341750665178271, + "nauc_ndcg_at_100_diff1": 0.11535802471187034, + "nauc_ndcg_at_100_max": -0.14740997248620716, + "nauc_ndcg_at_100_std": -0.06341750665178271, + "nauc_ndcg_at_10_diff1": 0.08167268516854659, + "nauc_ndcg_at_10_max": -0.11308549645360776, + "nauc_ndcg_at_10_std": -0.05170622944252826, + "nauc_ndcg_at_1_diff1": 0.15383540979942995, + "nauc_ndcg_at_1_max": -0.09134588524498606, + "nauc_ndcg_at_1_std": -0.06288060785518773, + "nauc_ndcg_at_20_diff1": 0.12317615651537264, + "nauc_ndcg_at_20_max": -0.16530409258087564, + "nauc_ndcg_at_20_std": -0.04941960930592955, + "nauc_ndcg_at_3_diff1": 0.11841235406595614, + "nauc_ndcg_at_3_max": -0.1478158585778922, + "nauc_ndcg_at_3_std": -0.1031131978248066, + "nauc_ndcg_at_5_diff1": 0.13690017610918653, + "nauc_ndcg_at_5_max": -0.1530868114394824, + "nauc_ndcg_at_5_std": -0.013708385632683178, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": -0.04973014649190272, + "nauc_precision_at_10_max": 0.020211477034916752, + "nauc_precision_at_10_std": 0.05105187796012856, + "nauc_precision_at_1_diff1": 0.15383540979942995, + "nauc_precision_at_1_max": -0.09134588524498606, + "nauc_precision_at_1_std": -0.06288060785518773, + "nauc_precision_at_20_diff1": 0.5541549953314721, + "nauc_precision_at_20_max": -1.1517273576096776, + "nauc_precision_at_20_std": 0.7222222222222204, + "nauc_precision_at_3_diff1": 0.1404571089344015, + "nauc_precision_at_3_max": -0.1641139804096174, + "nauc_precision_at_3_std": -0.104600771742357, + "nauc_precision_at_5_diff1": 0.19930254247597423, + "nauc_precision_at_5_max": -0.1689712501520617, + "nauc_precision_at_5_std": 0.16223997404809226, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": -0.049730146491904284, + "nauc_recall_at_10_max": 0.02021147703491618, + "nauc_recall_at_10_std": 0.05105187796012839, + "nauc_recall_at_1_diff1": 0.15383540979942995, + "nauc_recall_at_1_max": -0.09134588524498606, + "nauc_recall_at_1_std": -0.06288060785518773, + "nauc_recall_at_20_diff1": 0.5541549953314666, + "nauc_recall_at_20_max": -1.1517273576097098, + "nauc_recall_at_20_std": 0.7222222222222248, + "nauc_recall_at_3_diff1": 0.14045710893440153, + "nauc_recall_at_3_max": -0.1641139804096172, + "nauc_recall_at_3_std": -0.10460077174235681, + "nauc_recall_at_5_diff1": 0.1993025424759744, + "nauc_recall_at_5_max": -0.1689712501520623, + "nauc_recall_at_5_std": 0.16223997404809212, + "ndcg_at_1": 0.28, + "ndcg_at_10": 0.54728, + "ndcg_at_100": 0.59378, + "ndcg_at_1000": 0.59378, + "ndcg_at_20": 0.58962, + "ndcg_at_3": 0.46357, + "ndcg_at_5": 0.52123, + "precision_at_1": 0.28, + "precision_at_10": 0.082, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.049, + "precision_at_3": 0.2, + "precision_at_5": 0.148, + "recall_at_1": 0.28, + "recall_at_10": 0.82, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.98, + "recall_at_3": 0.6, + "recall_at_5": 0.74 + } + ], + "test_2048": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.64, + "map_at_1": 0.64, + "map_at_10": 0.76063, + "map_at_100": 0.7634, + "map_at_1000": 0.7634, + "map_at_20": 0.7634, + "map_at_3": 0.75, + "map_at_5": 0.75, + "mrr_at_1": 0.64, + "mrr_at_10": 0.7606349206349207, + "mrr_at_100": 0.7633968253968254, + "mrr_at_1000": 0.7633968253968254, + "mrr_at_20": 0.7633968253968254, + "mrr_at_3": 0.75, + "mrr_at_5": 0.75, + "nauc_map_at_1000_diff1": 0.6659897673927945, + "nauc_map_at_1000_max": 0.438015883604351, + "nauc_map_at_1000_std": -0.32353103697304714, + "nauc_map_at_100_diff1": 0.6659897673927945, + "nauc_map_at_100_max": 0.438015883604351, + "nauc_map_at_100_std": -0.32353103697304714, + "nauc_map_at_10_diff1": 0.6643661626981698, + "nauc_map_at_10_max": 0.4447904552039341, + "nauc_map_at_10_std": -0.3195361391174566, + "nauc_map_at_1_diff1": 0.6431217269127601, + "nauc_map_at_1_max": 0.3684378592412823, + "nauc_map_at_1_std": -0.3652446033976243, + "nauc_map_at_20_diff1": 0.6659897673927945, + "nauc_map_at_20_max": 0.438015883604351, + "nauc_map_at_20_std": -0.32353103697304714, + "nauc_map_at_3_diff1": 0.6763497267759563, + "nauc_map_at_3_max": 0.4640655737704917, + "nauc_map_at_3_std": -0.3349872495446268, + "nauc_map_at_5_diff1": 0.6763497267759563, + "nauc_map_at_5_max": 0.4640655737704917, + "nauc_map_at_5_std": -0.3349872495446268, + "nauc_mrr_at_1000_diff1": 0.6659897673927945, + "nauc_mrr_at_1000_max": 0.438015883604351, + "nauc_mrr_at_1000_std": -0.32353103697304714, + "nauc_mrr_at_100_diff1": 0.6659897673927945, + "nauc_mrr_at_100_max": 0.438015883604351, + "nauc_mrr_at_100_std": -0.32353103697304714, + "nauc_mrr_at_10_diff1": 0.6643661626981698, + "nauc_mrr_at_10_max": 0.4447904552039341, + "nauc_mrr_at_10_std": -0.3195361391174566, + "nauc_mrr_at_1_diff1": 0.6431217269127601, + "nauc_mrr_at_1_max": 0.3684378592412823, + "nauc_mrr_at_1_std": -0.3652446033976243, + "nauc_mrr_at_20_diff1": 0.6659897673927945, + "nauc_mrr_at_20_max": 0.438015883604351, + "nauc_mrr_at_20_std": -0.32353103697304714, + "nauc_mrr_at_3_diff1": 0.6763497267759563, + "nauc_mrr_at_3_max": 0.4640655737704917, + "nauc_mrr_at_3_std": -0.3349872495446268, + "nauc_mrr_at_5_diff1": 0.6763497267759563, + "nauc_mrr_at_5_max": 0.4640655737704917, + "nauc_mrr_at_5_std": -0.3349872495446268, + "nauc_ndcg_at_1000_diff1": 0.6684883589966267, + "nauc_ndcg_at_1000_max": 0.4492424304950073, + "nauc_ndcg_at_1000_std": -0.3218755830335738, + "nauc_ndcg_at_100_diff1": 0.6684883589966267, + "nauc_ndcg_at_100_max": 0.4492424304950073, + "nauc_ndcg_at_100_std": -0.3218755830335738, + "nauc_ndcg_at_10_diff1": 0.6606973995340676, + "nauc_ndcg_at_10_max": 0.47925721278476685, + "nauc_ndcg_at_10_std": -0.3036094638511953, + "nauc_ndcg_at_1_diff1": 0.6431217269127601, + "nauc_ndcg_at_1_max": 0.3684378592412823, + "nauc_ndcg_at_1_std": -0.3652446033976243, + "nauc_ndcg_at_20_diff1": 0.6684883589966267, + "nauc_ndcg_at_20_max": 0.4492424304950073, + "nauc_ndcg_at_20_std": -0.3218755830335738, + "nauc_ndcg_at_3_diff1": 0.6952817226938491, + "nauc_ndcg_at_3_max": 0.5244337105769169, + "nauc_ndcg_at_3_std": -0.348791547501269, + "nauc_ndcg_at_5_diff1": 0.6952817226938491, + "nauc_ndcg_at_5_max": 0.5244337105769169, + "nauc_ndcg_at_5_std": -0.348791547501269, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.5401493930905749, + "nauc_precision_at_10_max": 0.934640522875822, + "nauc_precision_at_10_std": -0.02427637721754168, + "nauc_precision_at_1_diff1": 0.6431217269127601, + "nauc_precision_at_1_max": 0.3684378592412823, + "nauc_precision_at_1_std": -0.3652446033976243, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.7959751829462268, + "nauc_precision_at_3_max": 0.8480750874960222, + "nauc_precision_at_3_std": -0.43573019408208663, + "nauc_precision_at_5_diff1": 0.7959751829462307, + "nauc_precision_at_5_max": 0.8480750874960236, + "nauc_precision_at_5_std": -0.4357301940820848, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.540149393090569, + "nauc_recall_at_10_max": 0.9346405228758167, + "nauc_recall_at_10_std": -0.024276377217550837, + "nauc_recall_at_1_diff1": 0.6431217269127601, + "nauc_recall_at_1_max": 0.3684378592412823, + "nauc_recall_at_1_std": -0.3652446033976243, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.7959751829462297, + "nauc_recall_at_3_max": 0.8480750874960229, + "nauc_recall_at_3_std": -0.435730194082087, + "nauc_recall_at_5_diff1": 0.7959751829462297, + "nauc_recall_at_5_max": 0.8480750874960229, + "nauc_recall_at_5_std": -0.435730194082087, + "ndcg_at_1": 0.64, + "ndcg_at_10": 0.8094, + "ndcg_at_100": 0.81952, + "ndcg_at_1000": 0.81952, + "ndcg_at_20": 0.81952, + "ndcg_at_3": 0.78357, + "ndcg_at_5": 0.78357, + "precision_at_1": 0.64, + "precision_at_10": 0.096, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.29333, + "precision_at_5": 0.176, + "recall_at_1": 0.64, + "recall_at_10": 0.96, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.88, + "recall_at_5": 0.88 + } + ], + "test_256": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.84, + "map_at_1": 0.84, + "map_at_10": 0.92, + "map_at_100": 0.92, + "map_at_1000": 0.92, + "map_at_20": 0.92, + "map_at_3": 0.92, + "map_at_5": 0.92, + "mrr_at_1": 0.84, + "mrr_at_10": 0.92, + "mrr_at_100": 0.92, + "mrr_at_1000": 0.92, + "mrr_at_20": 0.92, + "mrr_at_3": 0.92, + "mrr_at_5": 0.92, + "nauc_map_at_1000_diff1": 0.7704958271968593, + "nauc_map_at_1000_max": -0.07333087874325032, + "nauc_map_at_1000_std": -0.1825601374570458, + "nauc_map_at_100_diff1": 0.7704958271968593, + "nauc_map_at_100_max": -0.07333087874325032, + "nauc_map_at_100_std": -0.1825601374570458, + "nauc_map_at_10_diff1": 0.7704958271968593, + "nauc_map_at_10_max": -0.07333087874325032, + "nauc_map_at_10_std": -0.1825601374570458, + "nauc_map_at_1_diff1": 0.7704958271968577, + "nauc_map_at_1_max": -0.07333087874325026, + "nauc_map_at_1_std": -0.18256013745704477, + "nauc_map_at_20_diff1": 0.7704958271968593, + "nauc_map_at_20_max": -0.07333087874325032, + "nauc_map_at_20_std": -0.1825601374570458, + "nauc_map_at_3_diff1": 0.7704958271968593, + "nauc_map_at_3_max": -0.07333087874325032, + "nauc_map_at_3_std": -0.1825601374570458, + "nauc_map_at_5_diff1": 0.7704958271968593, + "nauc_map_at_5_max": -0.07333087874325032, + "nauc_map_at_5_std": -0.1825601374570458, + "nauc_mrr_at_1000_diff1": 0.7704958271968593, + "nauc_mrr_at_1000_max": -0.07333087874325032, + "nauc_mrr_at_1000_std": -0.1825601374570458, + "nauc_mrr_at_100_diff1": 0.7704958271968593, + "nauc_mrr_at_100_max": -0.07333087874325032, + "nauc_mrr_at_100_std": -0.1825601374570458, + "nauc_mrr_at_10_diff1": 0.7704958271968593, + "nauc_mrr_at_10_max": -0.07333087874325032, + "nauc_mrr_at_10_std": -0.1825601374570458, + "nauc_mrr_at_1_diff1": 0.7704958271968577, + "nauc_mrr_at_1_max": -0.07333087874325026, + "nauc_mrr_at_1_std": -0.18256013745704477, + "nauc_mrr_at_20_diff1": 0.7704958271968593, + "nauc_mrr_at_20_max": -0.07333087874325032, + "nauc_mrr_at_20_std": -0.1825601374570458, + "nauc_mrr_at_3_diff1": 0.7704958271968593, + "nauc_mrr_at_3_max": -0.07333087874325032, + "nauc_mrr_at_3_std": -0.1825601374570458, + "nauc_mrr_at_5_diff1": 0.7704958271968593, + "nauc_mrr_at_5_max": -0.07333087874325032, + "nauc_mrr_at_5_std": -0.1825601374570458, + "nauc_ndcg_at_1000_diff1": 0.7704958271968592, + "nauc_ndcg_at_1000_max": -0.07333087874324976, + "nauc_ndcg_at_1000_std": -0.18256013745704522, + "nauc_ndcg_at_100_diff1": 0.7704958271968592, + "nauc_ndcg_at_100_max": -0.07333087874324976, + "nauc_ndcg_at_100_std": -0.18256013745704522, + "nauc_ndcg_at_10_diff1": 0.7704958271968592, + "nauc_ndcg_at_10_max": -0.07333087874324976, + "nauc_ndcg_at_10_std": -0.18256013745704522, + "nauc_ndcg_at_1_diff1": 0.7704958271968577, + "nauc_ndcg_at_1_max": -0.07333087874325026, + "nauc_ndcg_at_1_std": -0.18256013745704477, + "nauc_ndcg_at_20_diff1": 0.7704958271968592, + "nauc_ndcg_at_20_max": -0.07333087874324976, + "nauc_ndcg_at_20_std": -0.18256013745704522, + "nauc_ndcg_at_3_diff1": 0.7704958271968592, + "nauc_ndcg_at_3_max": -0.07333087874324976, + "nauc_ndcg_at_3_std": -0.18256013745704522, + "nauc_ndcg_at_5_diff1": 0.7704958271968592, + "nauc_ndcg_at_5_max": -0.07333087874324976, + "nauc_ndcg_at_5_std": -0.18256013745704522, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.7704958271968577, + "nauc_precision_at_1_max": -0.07333087874325026, + "nauc_precision_at_1_std": -0.18256013745704477, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.7704958271968577, + "nauc_recall_at_1_max": -0.07333087874325026, + "nauc_recall_at_1_std": -0.18256013745704477, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.84, + "ndcg_at_10": 0.94095, + "ndcg_at_100": 0.94095, + "ndcg_at_1000": 0.94095, + "ndcg_at_20": 0.94095, + "ndcg_at_3": 0.94095, + "ndcg_at_5": 0.94095, + "precision_at_1": 0.84, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 0.84, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_32768": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.3, + "map_at_1": 0.3, + "map_at_10": 0.36769, + "map_at_100": 0.37935, + "map_at_1000": 0.37935, + "map_at_20": 0.37187, + "map_at_3": 0.34333, + "map_at_5": 0.36233, + "mrr_at_1": 0.3, + "mrr_at_10": 0.3676904761904762, + "mrr_at_100": 0.37878061344751435, + "mrr_at_1000": 0.37878061344751435, + "mrr_at_20": 0.3715862450243255, + "mrr_at_3": 0.34333333333333327, + "mrr_at_5": 0.36233333333333334, + "nauc_map_at_1000_diff1": 0.4464099965433418, + "nauc_map_at_1000_max": -0.0036852374983598773, + "nauc_map_at_1000_std": 0.33633319279675455, + "nauc_map_at_100_diff1": 0.4464099965433418, + "nauc_map_at_100_max": -0.0036852374983598773, + "nauc_map_at_100_std": 0.33633319279675455, + "nauc_map_at_10_diff1": 0.4475820701070957, + "nauc_map_at_10_max": -0.0030362863759158605, + "nauc_map_at_10_std": 0.34346459648596883, + "nauc_map_at_1_diff1": 0.44399405485745175, + "nauc_map_at_1_max": 0.020753952168625682, + "nauc_map_at_1_std": 0.32927982705039854, + "nauc_map_at_20_diff1": 0.4481383834920022, + "nauc_map_at_20_max": -0.005595505869845292, + "nauc_map_at_20_std": 0.3364528605030032, + "nauc_map_at_3_diff1": 0.42177503626527213, + "nauc_map_at_3_max": 0.04591690512828443, + "nauc_map_at_3_std": 0.3326087627479598, + "nauc_map_at_5_diff1": 0.43538425552558596, + "nauc_map_at_5_max": 0.006611573182471965, + "nauc_map_at_5_std": 0.3268432267992414, + "nauc_mrr_at_1000_diff1": 0.4454556487043541, + "nauc_mrr_at_1000_max": -0.005174562730196071, + "nauc_mrr_at_1000_std": 0.3393524089284785, + "nauc_mrr_at_100_diff1": 0.4454556487043541, + "nauc_mrr_at_100_max": -0.005174562730196071, + "nauc_mrr_at_100_std": 0.3393524089284785, + "nauc_mrr_at_10_diff1": 0.4475820701070957, + "nauc_mrr_at_10_max": -0.0030362863759158605, + "nauc_mrr_at_10_std": 0.34346459648596883, + "nauc_mrr_at_1_diff1": 0.44399405485745175, + "nauc_mrr_at_1_max": 0.020753952168625682, + "nauc_mrr_at_1_std": 0.32927982705039854, + "nauc_mrr_at_20_diff1": 0.4477512447856079, + "nauc_mrr_at_20_max": -0.008148490551017809, + "nauc_mrr_at_20_std": 0.33981600851395377, + "nauc_mrr_at_3_diff1": 0.42177503626527213, + "nauc_mrr_at_3_max": 0.04591690512828443, + "nauc_mrr_at_3_std": 0.3326087627479598, + "nauc_mrr_at_5_diff1": 0.43538425552558596, + "nauc_mrr_at_5_max": 0.006611573182471965, + "nauc_mrr_at_5_std": 0.3268432267992414, + "nauc_ndcg_at_1000_diff1": 0.4544628717039974, + "nauc_ndcg_at_1000_max": -0.020519816959366895, + "nauc_ndcg_at_1000_std": 0.3378235770934867, + "nauc_ndcg_at_100_diff1": 0.4544628717039974, + "nauc_ndcg_at_100_max": -0.020519816959366895, + "nauc_ndcg_at_100_std": 0.3378235770934867, + "nauc_ndcg_at_10_diff1": 0.46860076781355, + "nauc_ndcg_at_10_max": -0.03696183760661312, + "nauc_ndcg_at_10_std": 0.36346115268817475, + "nauc_ndcg_at_1_diff1": 0.44399405485745175, + "nauc_ndcg_at_1_max": 0.020753952168625682, + "nauc_ndcg_at_1_std": 0.32927982705039854, + "nauc_ndcg_at_20_diff1": 0.4706059020426094, + "nauc_ndcg_at_20_max": -0.04567981008780234, + "nauc_ndcg_at_20_std": 0.3368375219330409, + "nauc_ndcg_at_3_diff1": 0.41476447117334303, + "nauc_ndcg_at_3_max": 0.05614108896025905, + "nauc_ndcg_at_3_std": 0.3312371307254533, + "nauc_ndcg_at_5_diff1": 0.439562366221309, + "nauc_ndcg_at_5_max": -0.013711089668463664, + "nauc_ndcg_at_5_std": 0.3241811055568415, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.5452146471516401, + "nauc_precision_at_10_max": -0.154578595135753, + "nauc_precision_at_10_std": 0.4373469720877084, + "nauc_precision_at_1_diff1": 0.44399405485745175, + "nauc_precision_at_1_max": 0.020753952168625682, + "nauc_precision_at_1_std": 0.32927982705039854, + "nauc_precision_at_20_diff1": 0.5580288475491192, + "nauc_precision_at_20_max": -0.19633165974983344, + "nauc_precision_at_20_std": 0.328090083292219, + "nauc_precision_at_3_diff1": 0.395030895983522, + "nauc_precision_at_3_max": 0.08560762100926887, + "nauc_precision_at_3_std": 0.3266477857878475, + "nauc_precision_at_5_diff1": 0.45535619773473046, + "nauc_precision_at_5_max": -0.08079284428919263, + "nauc_precision_at_5_std": 0.3170847888218478, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.5452146471516406, + "nauc_recall_at_10_max": -0.15457859513575273, + "nauc_recall_at_10_std": 0.4373469720877087, + "nauc_recall_at_1_diff1": 0.44399405485745175, + "nauc_recall_at_1_max": 0.020753952168625682, + "nauc_recall_at_1_std": 0.32927982705039854, + "nauc_recall_at_20_diff1": 0.5580288475491194, + "nauc_recall_at_20_max": -0.19633165974983333, + "nauc_recall_at_20_std": 0.32809008329221906, + "nauc_recall_at_3_diff1": 0.39503089598352203, + "nauc_recall_at_3_max": 0.08560762100926869, + "nauc_recall_at_3_std": 0.3266477857878476, + "nauc_recall_at_5_diff1": 0.4553561977347303, + "nauc_recall_at_5_max": -0.08079284428919227, + "nauc_recall_at_5_std": 0.31708478882184804, + "ndcg_at_1": 0.3, + "ndcg_at_10": 0.40441, + "ndcg_at_100": 0.49127, + "ndcg_at_1000": 0.49127, + "ndcg_at_20": 0.41961, + "ndcg_at_3": 0.35786, + "ndcg_at_5": 0.39143, + "precision_at_1": 0.3, + "precision_at_10": 0.052, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.029, + "precision_at_3": 0.13333, + "precision_at_5": 0.096, + "recall_at_1": 0.3, + "recall_at_10": 0.52, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.58, + "recall_at_3": 0.4, + "recall_at_5": 0.48 + } + ], + "test_4096": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48, + "map_at_1": 0.48, + "map_at_10": 0.64078, + "map_at_100": 0.64402, + "map_at_1000": 0.64402, + "map_at_20": 0.64402, + "map_at_3": 0.59667, + "map_at_5": 0.62967, + "mrr_at_1": 0.48, + "mrr_at_10": 0.6407777777777778, + "mrr_at_100": 0.644024531024531, + "mrr_at_1000": 0.644024531024531, + "mrr_at_20": 0.644024531024531, + "mrr_at_3": 0.5966666666666667, + "mrr_at_5": 0.6296666666666666, + "nauc_map_at_1000_diff1": 0.48074717137776013, + "nauc_map_at_1000_max": -0.031425100957077276, + "nauc_map_at_1000_std": 0.011021454366977775, + "nauc_map_at_100_diff1": 0.48074717137776013, + "nauc_map_at_100_max": -0.031425100957077276, + "nauc_map_at_100_std": 0.011021454366977775, + "nauc_map_at_10_diff1": 0.48367169077698985, + "nauc_map_at_10_max": -0.020608264023183175, + "nauc_map_at_10_std": -0.00014186605358405054, + "nauc_map_at_1_diff1": 0.5196358632370064, + "nauc_map_at_1_max": -0.05210648883243348, + "nauc_map_at_1_std": 0.0392187996189266, + "nauc_map_at_20_diff1": 0.48074717137776013, + "nauc_map_at_20_max": -0.031425100957077276, + "nauc_map_at_20_std": 0.011021454366977775, + "nauc_map_at_3_diff1": 0.48400280061198553, + "nauc_map_at_3_max": -0.02978502709851436, + "nauc_map_at_3_std": 0.019443508025828015, + "nauc_map_at_5_diff1": 0.4558065560211764, + "nauc_map_at_5_max": -0.03683872520103921, + "nauc_map_at_5_std": 0.008525091482473976, + "nauc_mrr_at_1000_diff1": 0.48074717137776013, + "nauc_mrr_at_1000_max": -0.031425100957077276, + "nauc_mrr_at_1000_std": 0.011021454366977775, + "nauc_mrr_at_100_diff1": 0.48074717137776013, + "nauc_mrr_at_100_max": -0.031425100957077276, + "nauc_mrr_at_100_std": 0.011021454366977775, + "nauc_mrr_at_10_diff1": 0.48367169077698985, + "nauc_mrr_at_10_max": -0.020608264023183175, + "nauc_mrr_at_10_std": -0.00014186605358405054, + "nauc_mrr_at_1_diff1": 0.5196358632370064, + "nauc_mrr_at_1_max": -0.05210648883243348, + "nauc_mrr_at_1_std": 0.0392187996189266, + "nauc_mrr_at_20_diff1": 0.48074717137776013, + "nauc_mrr_at_20_max": -0.031425100957077276, + "nauc_mrr_at_20_std": 0.011021454366977775, + "nauc_mrr_at_3_diff1": 0.48400280061198553, + "nauc_mrr_at_3_max": -0.02978502709851436, + "nauc_mrr_at_3_std": 0.019443508025828015, + "nauc_mrr_at_5_diff1": 0.4558065560211764, + "nauc_mrr_at_5_max": -0.03683872520103921, + "nauc_mrr_at_5_std": 0.008525091482473976, + "nauc_ndcg_at_1000_diff1": 0.4772092627934861, + "nauc_ndcg_at_1000_max": -0.024210749637897166, + "nauc_ndcg_at_1000_std": 0.007841187939444163, + "nauc_ndcg_at_100_diff1": 0.4772092627934861, + "nauc_ndcg_at_100_max": -0.024210749637897166, + "nauc_ndcg_at_100_std": 0.007841187939444163, + "nauc_ndcg_at_10_diff1": 0.4882075172927644, + "nauc_ndcg_at_10_max": 0.020642774301884773, + "nauc_ndcg_at_10_std": -0.03694783777246808, + "nauc_ndcg_at_1_diff1": 0.5196358632370064, + "nauc_ndcg_at_1_max": -0.05210648883243348, + "nauc_ndcg_at_1_std": 0.0392187996189266, + "nauc_ndcg_at_20_diff1": 0.4772092627934861, + "nauc_ndcg_at_20_max": -0.024210749637897166, + "nauc_ndcg_at_20_std": 0.007841187939444163, + "nauc_ndcg_at_3_diff1": 0.4748614182223308, + "nauc_ndcg_at_3_max": -0.010837772733904648, + "nauc_ndcg_at_3_std": 0.022335979097033068, + "nauc_ndcg_at_5_diff1": 0.4155452556699317, + "nauc_ndcg_at_5_max": -0.016607042542615483, + "nauc_ndcg_at_5_std": -0.005579570881067056, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.6790382819794623, + "nauc_precision_at_10_max": 0.8611111111111167, + "nauc_precision_at_10_std": -0.8576097105508743, + "nauc_precision_at_1_diff1": 0.5196358632370064, + "nauc_precision_at_1_max": -0.05210648883243348, + "nauc_precision_at_1_std": 0.0392187996189266, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.4423989294837996, + "nauc_precision_at_3_max": 0.06431207169214546, + "nauc_precision_at_3_std": 0.03734641742021805, + "nauc_precision_at_5_diff1": 0.13044861597200266, + "nauc_precision_at_5_max": 0.13235762010817767, + "nauc_precision_at_5_std": -0.1097677378300969, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.6790382819794599, + "nauc_recall_at_10_max": 0.8611111111111124, + "nauc_recall_at_10_std": -0.8576097105508831, + "nauc_recall_at_1_diff1": 0.5196358632370064, + "nauc_recall_at_1_max": -0.05210648883243348, + "nauc_recall_at_1_std": 0.0392187996189266, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.44239892948380066, + "nauc_recall_at_3_max": 0.0643120716921455, + "nauc_recall_at_3_std": 0.03734641742021736, + "nauc_recall_at_5_diff1": 0.13044861597200041, + "nauc_recall_at_5_max": 0.1323576201081771, + "nauc_recall_at_5_std": -0.10976773783009851, + "ndcg_at_1": 0.48, + "ndcg_at_10": 0.7184, + "ndcg_at_100": 0.7291, + "ndcg_at_1000": 0.7291, + "ndcg_at_20": 0.7291, + "ndcg_at_3": 0.63357, + "ndcg_at_5": 0.69211, + "precision_at_1": 0.48, + "precision_at_10": 0.096, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.24667, + "precision_at_5": 0.176, + "recall_at_1": 0.48, + "recall_at_10": 0.96, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.74, + "recall_at_5": 0.88 + } + ], + "test_512": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8, + "map_at_1": 0.8, + "map_at_10": 0.888, + "map_at_100": 0.888, + "map_at_1000": 0.888, + "map_at_20": 0.888, + "map_at_3": 0.88, + "map_at_5": 0.888, + "mrr_at_1": 0.8, + "mrr_at_10": 0.888, + "mrr_at_100": 0.888, + "mrr_at_1000": 0.888, + "mrr_at_20": 0.888, + "mrr_at_3": 0.88, + "mrr_at_5": 0.888, + "nauc_map_at_1000_diff1": 0.6593951412989592, + "nauc_map_at_1000_max": 0.026055315532260752, + "nauc_map_at_1000_std": -1.1468765493306892, + "nauc_map_at_100_diff1": 0.6593951412989592, + "nauc_map_at_100_max": 0.026055315532260752, + "nauc_map_at_100_std": -1.1468765493306892, + "nauc_map_at_10_diff1": 0.6593951412989592, + "nauc_map_at_10_max": 0.026055315532260752, + "nauc_map_at_10_std": -1.1468765493306892, + "nauc_map_at_1_diff1": 0.6803696303696307, + "nauc_map_at_1_max": 0.05769230769230764, + "nauc_map_at_1_std": -1.070729270729271, + "nauc_map_at_20_diff1": 0.6593951412989592, + "nauc_map_at_20_max": 0.026055315532260752, + "nauc_map_at_20_std": -1.1468765493306892, + "nauc_map_at_3_diff1": 0.6478768926925608, + "nauc_map_at_3_max": 0.00868169848584675, + "nauc_map_at_3_std": -1.188693219223172, + "nauc_map_at_5_diff1": 0.6593951412989592, + "nauc_map_at_5_max": 0.026055315532260752, + "nauc_map_at_5_std": -1.1468765493306892, + "nauc_mrr_at_1000_diff1": 0.6593951412989592, + "nauc_mrr_at_1000_max": 0.026055315532260752, + "nauc_mrr_at_1000_std": -1.1468765493306892, + "nauc_mrr_at_100_diff1": 0.6593951412989592, + "nauc_mrr_at_100_max": 0.026055315532260752, + "nauc_mrr_at_100_std": -1.1468765493306892, + "nauc_mrr_at_10_diff1": 0.6593951412989592, + "nauc_mrr_at_10_max": 0.026055315532260752, + "nauc_mrr_at_10_std": -1.1468765493306892, + "nauc_mrr_at_1_diff1": 0.6803696303696307, + "nauc_mrr_at_1_max": 0.05769230769230764, + "nauc_mrr_at_1_std": -1.070729270729271, + "nauc_mrr_at_20_diff1": 0.6593951412989592, + "nauc_mrr_at_20_max": 0.026055315532260752, + "nauc_mrr_at_20_std": -1.1468765493306892, + "nauc_mrr_at_3_diff1": 0.6478768926925608, + "nauc_mrr_at_3_max": 0.00868169848584675, + "nauc_mrr_at_3_std": -1.188693219223172, + "nauc_mrr_at_5_diff1": 0.6593951412989592, + "nauc_mrr_at_5_max": 0.026055315532260752, + "nauc_mrr_at_5_std": -1.1468765493306892, + "nauc_ndcg_at_1000_diff1": 0.6575170096312764, + "nauc_ndcg_at_1000_max": 0.023222424745469407, + "nauc_ndcg_at_1000_std": -1.1536950522685785, + "nauc_ndcg_at_100_diff1": 0.6575170096312764, + "nauc_ndcg_at_100_max": 0.023222424745469407, + "nauc_ndcg_at_100_std": -1.1536950522685785, + "nauc_ndcg_at_10_diff1": 0.6575170096312764, + "nauc_ndcg_at_10_max": 0.023222424745469407, + "nauc_ndcg_at_10_std": -1.1536950522685785, + "nauc_ndcg_at_1_diff1": 0.6803696303696307, + "nauc_ndcg_at_1_max": 0.05769230769230764, + "nauc_ndcg_at_1_std": -1.070729270729271, + "nauc_ndcg_at_20_diff1": 0.6575170096312764, + "nauc_ndcg_at_20_max": 0.023222424745469407, + "nauc_ndcg_at_20_std": -1.1536950522685785, + "nauc_ndcg_at_3_diff1": 0.6309975765138996, + "nauc_ndcg_at_3_max": -0.016778315906493677, + "nauc_ndcg_at_3_std": -1.2499730905624584, + "nauc_ndcg_at_5_diff1": 0.6575170096312764, + "nauc_ndcg_at_5_max": 0.023222424745469407, + "nauc_ndcg_at_5_std": -1.1536950522685785, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.6803696303696307, + "nauc_precision_at_1_max": 0.05769230769230764, + "nauc_precision_at_1_std": -1.070729270729271, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.4960317460317487, + "nauc_precision_at_3_max": -0.2203548085901082, + "nauc_precision_at_3_std": -1.7399626517273643, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.6803696303696307, + "nauc_recall_at_1_max": 0.05769230769230764, + "nauc_recall_at_1_std": -1.070729270729271, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.4960317460317471, + "nauc_recall_at_3_max": -0.22035480859010095, + "nauc_recall_at_3_std": -1.7399626517273536, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.8, + "ndcg_at_10": 0.91642, + "ndcg_at_100": 0.91642, + "ndcg_at_1000": 0.91642, + "ndcg_at_20": 0.91642, + "ndcg_at_3": 0.90095, + "ndcg_at_5": 0.91642, + "precision_at_1": 0.8, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.32, + "precision_at_5": 0.2, + "recall_at_1": 0.8, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.96, + "recall_at_5": 1.0 + } + ], + "test_8192": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.2, + "map_at_1": 0.2, + "map_at_10": 0.35863, + "map_at_100": 0.37078, + "map_at_1000": 0.37078, + "map_at_20": 0.37009, + "map_at_3": 0.28667, + "map_at_5": 0.32367, + "mrr_at_1": 0.2, + "mrr_at_10": 0.3586349206349206, + "mrr_at_100": 0.37077912126187984, + "mrr_at_1000": 0.37077912126187984, + "mrr_at_20": 0.370089466089466, + "mrr_at_3": 0.28666666666666674, + "mrr_at_5": 0.32366666666666677, + "nauc_map_at_1000_diff1": 0.5580044161987427, + "nauc_map_at_1000_max": 0.003351304845841549, + "nauc_map_at_1000_std": 0.5117818018873851, + "nauc_map_at_100_diff1": 0.5580044161987427, + "nauc_map_at_100_max": 0.003351304845841549, + "nauc_map_at_100_std": 0.5117818018873851, + "nauc_map_at_10_diff1": 0.5433222562930573, + "nauc_map_at_10_max": -0.008374903972883346, + "nauc_map_at_10_std": 0.5147148024672304, + "nauc_map_at_1_diff1": 0.7057152082660642, + "nauc_map_at_1_max": 0.07313529221827568, + "nauc_map_at_1_std": 0.5618663222473362, + "nauc_map_at_20_diff1": 0.556929235557146, + "nauc_map_at_20_max": 0.0019509641952703298, + "nauc_map_at_20_std": 0.512664974826674, + "nauc_map_at_3_diff1": 0.524511566000525, + "nauc_map_at_3_max": 0.04203787002943734, + "nauc_map_at_3_std": 0.4662832382858425, + "nauc_map_at_5_diff1": 0.5448662315100441, + "nauc_map_at_5_max": -0.017070223778436342, + "nauc_map_at_5_std": 0.5261141385311243, + "nauc_mrr_at_1000_diff1": 0.5580044161987427, + "nauc_mrr_at_1000_max": 0.003351304845841549, + "nauc_mrr_at_1000_std": 0.5117818018873851, + "nauc_mrr_at_100_diff1": 0.5580044161987427, + "nauc_mrr_at_100_max": 0.003351304845841549, + "nauc_mrr_at_100_std": 0.5117818018873851, + "nauc_mrr_at_10_diff1": 0.5433222562930573, + "nauc_mrr_at_10_max": -0.008374903972883346, + "nauc_mrr_at_10_std": 0.5147148024672304, + "nauc_mrr_at_1_diff1": 0.7057152082660642, + "nauc_mrr_at_1_max": 0.07313529221827568, + "nauc_mrr_at_1_std": 0.5618663222473362, + "nauc_mrr_at_20_diff1": 0.556929235557146, + "nauc_mrr_at_20_max": 0.0019509641952703298, + "nauc_mrr_at_20_std": 0.512664974826674, + "nauc_mrr_at_3_diff1": 0.524511566000525, + "nauc_mrr_at_3_max": 0.04203787002943734, + "nauc_mrr_at_3_std": 0.4662832382858425, + "nauc_mrr_at_5_diff1": 0.5448662315100441, + "nauc_mrr_at_5_max": -0.017070223778436342, + "nauc_mrr_at_5_std": 0.5261141385311243, + "nauc_ndcg_at_1000_diff1": 0.5435227000208164, + "nauc_ndcg_at_1000_max": -0.00906643274222386, + "nauc_ndcg_at_1000_std": 0.5116581181242461, + "nauc_ndcg_at_100_diff1": 0.5435227000208164, + "nauc_ndcg_at_100_max": -0.00906643274222386, + "nauc_ndcg_at_100_std": 0.5116581181242461, + "nauc_ndcg_at_10_diff1": 0.48560214494775156, + "nauc_ndcg_at_10_max": -0.05293649308584778, + "nauc_ndcg_at_10_std": 0.5196582547509481, + "nauc_ndcg_at_1_diff1": 0.7057152082660642, + "nauc_ndcg_at_1_max": 0.07313529221827568, + "nauc_ndcg_at_1_std": 0.5618663222473362, + "nauc_ndcg_at_20_diff1": 0.5359187574586686, + "nauc_ndcg_at_20_max": -0.019086440607030422, + "nauc_ndcg_at_20_std": 0.518121355660027, + "nauc_ndcg_at_3_diff1": 0.4722700267755813, + "nauc_ndcg_at_3_max": 0.034231471489964964, + "nauc_ndcg_at_3_std": 0.44116376053900125, + "nauc_ndcg_at_5_diff1": 0.5060990562789037, + "nauc_ndcg_at_5_max": -0.0725474955214473, + "nauc_ndcg_at_5_std": 0.5489011947717592, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.14641629847815416, + "nauc_precision_at_10_max": -0.3036941580756014, + "nauc_precision_at_10_std": 0.5548600883652438, + "nauc_precision_at_1_diff1": 0.7057152082660642, + "nauc_precision_at_1_max": 0.07313529221827568, + "nauc_precision_at_1_std": 0.5618663222473362, + "nauc_precision_at_20_diff1": 0.12278244631187159, + "nauc_precision_at_20_max": -0.563492063492042, + "nauc_precision_at_20_std": 0.8692810457516293, + "nauc_precision_at_3_diff1": 0.33766233766233755, + "nauc_precision_at_3_max": 0.014438655157218174, + "nauc_precision_at_3_std": 0.3771158980739817, + "nauc_precision_at_5_diff1": 0.4021243869170269, + "nauc_precision_at_5_max": -0.24384014859099776, + "nauc_precision_at_5_std": 0.6305250021766264, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.1464162984781539, + "nauc_recall_at_10_max": -0.3036941580756018, + "nauc_recall_at_10_std": 0.5548600883652421, + "nauc_recall_at_1_diff1": 0.7057152082660642, + "nauc_recall_at_1_max": 0.07313529221827568, + "nauc_recall_at_1_std": 0.5618663222473362, + "nauc_recall_at_20_diff1": 0.12278244631186093, + "nauc_recall_at_20_max": -0.5634920634920563, + "nauc_recall_at_20_std": 0.8692810457516332, + "nauc_recall_at_3_diff1": 0.33766233766233755, + "nauc_recall_at_3_max": 0.014438655157217898, + "nauc_recall_at_3_std": 0.37711589807398194, + "nauc_recall_at_5_diff1": 0.402124386917027, + "nauc_recall_at_5_max": -0.24384014859099729, + "nauc_recall_at_5_std": 0.6305250021766263, + "ndcg_at_1": 0.2, + "ndcg_at_10": 0.47104, + "ndcg_at_100": 0.51267, + "ndcg_at_1000": 0.51267, + "ndcg_at_20": 0.50859, + "ndcg_at_3": 0.32047, + "ndcg_at_5": 0.38675, + "precision_at_1": 0.2, + "precision_at_10": 0.084, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.049, + "precision_at_3": 0.14, + "precision_at_5": 0.116, + "recall_at_1": 0.2, + "recall_at_10": 0.84, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.98, + "recall_at_3": 0.42, + "recall_at_5": 0.58 + } + ] + }, + "task_name": "LEMBNeedleRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LEMBPasskeyRetrieval.json b/results/voyageai__voyage-3-lite/no_revision_available/LEMBPasskeyRetrieval.json new file mode 100644 index 000000000..49ca1d6b9 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LEMBPasskeyRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 44.324267625808716, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test_1024": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_16384": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.48, + "map_at_1": 0.48, + "map_at_10": 0.48333, + "map_at_100": 0.49895, + "map_at_1000": 0.49895, + "map_at_20": 0.48741, + "map_at_3": 0.48, + "map_at_5": 0.48, + "mrr_at_1": 0.48, + "mrr_at_10": 0.4833333333333333, + "mrr_at_100": 0.49895196805731706, + "mrr_at_1000": 0.49895196805731706, + "mrr_at_20": 0.4874114774114775, + "mrr_at_3": 0.48, + "mrr_at_5": 0.48, + "nauc_map_at_1000_diff1": 0.9895947355824045, + "nauc_map_at_1000_max": 0.718471646814206, + "nauc_map_at_1000_std": 0.4750002222684473, + "nauc_map_at_100_diff1": 0.9895947355824045, + "nauc_map_at_100_max": 0.718471646814206, + "nauc_map_at_100_std": 0.4750002222684473, + "nauc_map_at_10_diff1": 0.9929393062945708, + "nauc_map_at_10_max": 0.7191221601571984, + "nauc_map_at_10_std": 0.47777360740070196, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_1_max": 0.7265004763416956, + "nauc_map_at_1_std": 0.4840425531914891, + "nauc_map_at_20_diff1": 0.9900795154790092, + "nauc_map_at_20_max": 0.7144062183346958, + "nauc_map_at_20_std": 0.47757384844169454, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_3_max": 0.7265004763416956, + "nauc_map_at_3_std": 0.4840425531914891, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_5_max": 0.7265004763416956, + "nauc_map_at_5_std": 0.4840425531914891, + "nauc_mrr_at_1000_diff1": 0.9895947355824045, + "nauc_mrr_at_1000_max": 0.718471646814206, + "nauc_mrr_at_1000_std": 0.4750002222684473, + "nauc_mrr_at_100_diff1": 0.9895947355824045, + "nauc_mrr_at_100_max": 0.718471646814206, + "nauc_mrr_at_100_std": 0.4750002222684473, + "nauc_mrr_at_10_diff1": 0.9929393062945708, + "nauc_mrr_at_10_max": 0.7191221601571984, + "nauc_mrr_at_10_std": 0.47777360740070196, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_1_max": 0.7265004763416956, + "nauc_mrr_at_1_std": 0.4840425531914891, + "nauc_mrr_at_20_diff1": 0.9900795154790092, + "nauc_mrr_at_20_max": 0.7144062183346958, + "nauc_mrr_at_20_std": 0.47757384844169454, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_3_max": 0.7265004763416956, + "nauc_mrr_at_3_std": 0.4840425531914891, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_5_max": 0.7265004763416956, + "nauc_mrr_at_5_std": 0.4840425531914891, + "nauc_ndcg_at_1000_diff1": 0.9822324610842013, + "nauc_ndcg_at_1000_max": 0.7147182121670457, + "nauc_ndcg_at_1000_std": 0.46773964360034154, + "nauc_ndcg_at_100_diff1": 0.9822324610842013, + "nauc_ndcg_at_100_max": 0.7147182121670457, + "nauc_ndcg_at_100_std": 0.46773964360034154, + "nauc_ndcg_at_10_diff1": 0.9848896244894796, + "nauc_ndcg_at_10_max": 0.7107103666393906, + "nauc_ndcg_at_10_std": 0.47062657327423874, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_1_max": 0.7265004763416956, + "nauc_ndcg_at_1_std": 0.4840425531914891, + "nauc_ndcg_at_20_diff1": 0.9730368231387787, + "nauc_ndcg_at_20_max": 0.6928395494155564, + "nauc_ndcg_at_20_std": 0.46706008279844324, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.7265004763416956, + "nauc_ndcg_at_3_std": 0.4840425531914891, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.7265004763416956, + "nauc_ndcg_at_5_std": 0.4840425531914891, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.9573884077281813, + "nauc_precision_at_10_max": 0.6819720186542304, + "nauc_precision_at_10_std": 0.44620919387075264, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_1_max": 0.7265004763416956, + "nauc_precision_at_1_std": 0.4840425531914891, + "nauc_precision_at_20_diff1": 0.9036452418805356, + "nauc_precision_at_20_max": 0.6049852373381781, + "nauc_precision_at_20_std": 0.42377356347944567, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 0.7265004763416956, + "nauc_precision_at_3_std": 0.48404255319148953, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 0.7265004763416956, + "nauc_precision_at_5_std": 0.48404255319148914, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.957388407728181, + "nauc_recall_at_10_max": 0.6819720186542302, + "nauc_recall_at_10_std": 0.4462091938707527, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_1_max": 0.7265004763416956, + "nauc_recall_at_1_std": 0.4840425531914891, + "nauc_recall_at_20_diff1": 0.9036452418805356, + "nauc_recall_at_20_max": 0.6049852373381781, + "nauc_recall_at_20_std": 0.42377356347944556, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_3_max": 0.7265004763416956, + "nauc_recall_at_3_std": 0.4840425531914891, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_5_max": 0.7265004763416956, + "nauc_recall_at_5_std": 0.4840425531914891, + "ndcg_at_1": 0.48, + "ndcg_at_10": 0.48712, + "ndcg_at_100": 0.58481, + "ndcg_at_1000": 0.58481, + "ndcg_at_20": 0.5022, + "ndcg_at_3": 0.48, + "ndcg_at_5": 0.48, + "precision_at_1": 0.48, + "precision_at_10": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.028, + "precision_at_3": 0.16, + "precision_at_5": 0.096, + "recall_at_1": 0.48, + "recall_at_10": 0.5, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.56, + "recall_at_3": 0.48, + "recall_at_5": 0.48 + } + ], + "test_2048": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_256": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_32768": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.38, + "map_at_1": 0.38, + "map_at_10": 0.38444, + "map_at_100": 0.39784, + "map_at_1000": 0.39784, + "map_at_20": 0.38611, + "map_at_3": 0.38, + "map_at_5": 0.38, + "mrr_at_1": 0.38, + "mrr_at_10": 0.3844444444444444, + "mrr_at_100": 0.3976604001440868, + "mrr_at_1000": 0.3976604001440868, + "mrr_at_20": 0.38611111111111107, + "mrr_at_3": 0.38, + "mrr_at_5": 0.38, + "nauc_map_at_1000_diff1": 0.9861182570526851, + "nauc_map_at_1000_max": 0.7917724119380407, + "nauc_map_at_1000_std": 0.37386674929966895, + "nauc_map_at_100_diff1": 0.9861182570526851, + "nauc_map_at_100_max": 0.7917724119380407, + "nauc_map_at_100_std": 0.37386674929966895, + "nauc_map_at_10_diff1": 0.992109388759127, + "nauc_map_at_10_max": 0.7976677512768469, + "nauc_map_at_10_std": 0.37342830168523583, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_1_max": 0.804590232110831, + "nauc_map_at_1_std": 0.38234606087385936, + "nauc_map_at_20_diff1": 0.9880338423650654, + "nauc_map_at_20_max": 0.7934814087263614, + "nauc_map_at_20_std": 0.3719793017033124, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_3_max": 0.804590232110831, + "nauc_map_at_3_std": 0.38234606087385936, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_5_max": 0.804590232110831, + "nauc_map_at_5_std": 0.38234606087385936, + "nauc_mrr_at_1000_diff1": 0.986055905622642, + "nauc_mrr_at_1000_max": 0.7921973491438137, + "nauc_mrr_at_1000_std": 0.37477355295480774, + "nauc_mrr_at_100_diff1": 0.986055905622642, + "nauc_mrr_at_100_max": 0.7921973491438137, + "nauc_mrr_at_100_std": 0.37477355295480774, + "nauc_mrr_at_10_diff1": 0.992109388759127, + "nauc_mrr_at_10_max": 0.7976677512768469, + "nauc_mrr_at_10_std": 0.37342830168523583, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_1_max": 0.804590232110831, + "nauc_mrr_at_1_std": 0.38234606087385936, + "nauc_mrr_at_20_diff1": 0.9880338423650654, + "nauc_mrr_at_20_max": 0.7934814087263614, + "nauc_mrr_at_20_std": 0.3719793017033124, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_3_max": 0.804590232110831, + "nauc_mrr_at_3_std": 0.38234606087385936, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_5_max": 0.804590232110831, + "nauc_mrr_at_5_std": 0.38234606087385936, + "nauc_ndcg_at_1000_diff1": 0.9735301454460804, + "nauc_ndcg_at_1000_max": 0.7801207126201983, + "nauc_ndcg_at_1000_std": 0.3675806856025691, + "nauc_ndcg_at_100_diff1": 0.9735301454460804, + "nauc_ndcg_at_100_max": 0.7801207126201983, + "nauc_ndcg_at_100_std": 0.3675806856025691, + "nauc_ndcg_at_10_diff1": 0.978633170164697, + "nauc_ndcg_at_10_max": 0.7858449834218939, + "nauc_ndcg_at_10_std": 0.35819783746265843, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_1_max": 0.804590232110831, + "nauc_ndcg_at_1_std": 0.38234606087385936, + "nauc_ndcg_at_20_diff1": 0.9653817200620957, + "nauc_ndcg_at_20_max": 0.772237019886636, + "nauc_ndcg_at_20_std": 0.3534671765422155, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.804590232110831, + "nauc_ndcg_at_3_std": 0.38234606087385936, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.804590232110831, + "nauc_ndcg_at_5_std": 0.38234606087385936, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.9291546776576715, + "nauc_precision_at_10_max": 0.7424372035150479, + "nauc_precision_at_10_std": 0.3022785597635896, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_1_max": 0.804590232110831, + "nauc_precision_at_1_std": 0.38234606087385936, + "nauc_precision_at_20_diff1": 0.8796262462807329, + "nauc_precision_at_20_max": 0.6916270814845746, + "nauc_precision_at_20_std": 0.28433470793965654, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 0.804590232110831, + "nauc_precision_at_3_std": 0.38234606087385986, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 0.8045902321108308, + "nauc_precision_at_5_std": 0.3823460608738593, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.9291546776576715, + "nauc_recall_at_10_max": 0.7424372035150475, + "nauc_recall_at_10_std": 0.30227855976358947, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_1_max": 0.804590232110831, + "nauc_recall_at_1_std": 0.38234606087385936, + "nauc_recall_at_20_diff1": 0.8796262462807325, + "nauc_recall_at_20_max": 0.6916270814845747, + "nauc_recall_at_20_std": 0.28433470793965643, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_3_max": 0.804590232110831, + "nauc_recall_at_3_std": 0.38234606087385936, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_5_max": 0.804590232110831, + "nauc_recall_at_5_std": 0.38234606087385936, + "ndcg_at_1": 0.38, + "ndcg_at_10": 0.39204, + "ndcg_at_100": 0.49551, + "ndcg_at_1000": 0.49551, + "ndcg_at_20": 0.39745, + "ndcg_at_3": 0.38, + "ndcg_at_5": 0.38, + "precision_at_1": 0.38, + "precision_at_10": 0.042, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.022, + "precision_at_3": 0.12667, + "precision_at_5": 0.076, + "recall_at_1": 0.38, + "recall_at_10": 0.42, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.44, + "recall_at_3": 0.38, + "recall_at_5": 0.38 + } + ], + "test_4096": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_512": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_8192": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.92, + "map_at_1": 0.92, + "map_at_10": 0.94, + "map_at_100": 0.94193, + "map_at_1000": 0.94193, + "map_at_20": 0.94143, + "map_at_3": 0.94, + "map_at_5": 0.94, + "mrr_at_1": 0.92, + "mrr_at_10": 0.94, + "mrr_at_100": 0.9419285714285713, + "mrr_at_1000": 0.9419285714285713, + "mrr_at_20": 0.9414285714285714, + "mrr_at_3": 0.94, + "mrr_at_5": 0.94, + "nauc_map_at_1000_diff1": 0.977489970978142, + "nauc_map_at_1000_max": 0.4556150463465409, + "nauc_map_at_1000_std": 0.5403475043153783, + "nauc_map_at_100_diff1": 0.977489970978142, + "nauc_map_at_100_max": 0.4556150463465409, + "nauc_map_at_100_std": 0.5403475043153783, + "nauc_map_at_10_diff1": 0.9782135076252712, + "nauc_map_at_10_max": 0.46335200746965377, + "nauc_map_at_10_std": 0.5445066915655148, + "nauc_map_at_1_diff1": 0.9673202614379083, + "nauc_map_at_1_max": 0.48786181139122375, + "nauc_map_at_1_std": 0.4282212885154062, + "nauc_map_at_20_diff1": 0.9776821297624728, + "nauc_map_at_20_max": 0.4502630320420843, + "nauc_map_at_20_std": 0.5442713670834188, + "nauc_map_at_3_diff1": 0.9782135076252712, + "nauc_map_at_3_max": 0.46335200746965377, + "nauc_map_at_3_std": 0.5445066915655148, + "nauc_map_at_5_diff1": 0.9782135076252712, + "nauc_map_at_5_max": 0.46335200746965377, + "nauc_map_at_5_std": 0.5445066915655148, + "nauc_mrr_at_1000_diff1": 0.977489970978142, + "nauc_mrr_at_1000_max": 0.4556150463465409, + "nauc_mrr_at_1000_std": 0.5403475043153783, + "nauc_mrr_at_100_diff1": 0.977489970978142, + "nauc_mrr_at_100_max": 0.4556150463465409, + "nauc_mrr_at_100_std": 0.5403475043153783, + "nauc_mrr_at_10_diff1": 0.9782135076252712, + "nauc_mrr_at_10_max": 0.46335200746965377, + "nauc_mrr_at_10_std": 0.5445066915655148, + "nauc_mrr_at_1_diff1": 0.9673202614379083, + "nauc_mrr_at_1_max": 0.48786181139122375, + "nauc_mrr_at_1_std": 0.4282212885154062, + "nauc_mrr_at_20_diff1": 0.9776821297624728, + "nauc_mrr_at_20_max": 0.4502630320420843, + "nauc_mrr_at_20_std": 0.5442713670834188, + "nauc_mrr_at_3_diff1": 0.9782135076252712, + "nauc_mrr_at_3_max": 0.46335200746965377, + "nauc_mrr_at_3_std": 0.5445066915655148, + "nauc_mrr_at_5_diff1": 0.9782135076252712, + "nauc_mrr_at_5_max": 0.46335200746965377, + "nauc_mrr_at_5_std": 0.5445066915655148, + "nauc_ndcg_at_1000_diff1": 0.9789832955947447, + "nauc_ndcg_at_1000_max": 0.44393777111139726, + "nauc_ndcg_at_1000_std": 0.5594545543144673, + "nauc_ndcg_at_100_diff1": 0.9789832955947447, + "nauc_ndcg_at_100_max": 0.44393777111139726, + "nauc_ndcg_at_100_std": 0.5594545543144673, + "nauc_ndcg_at_10_diff1": 0.9823805693012515, + "nauc_ndcg_at_10_max": 0.45397611869869464, + "nauc_ndcg_at_10_std": 0.5889900749566161, + "nauc_ndcg_at_1_diff1": 0.9673202614379083, + "nauc_ndcg_at_1_max": 0.48786181139122375, + "nauc_ndcg_at_1_std": 0.4282212885154062, + "nauc_ndcg_at_20_diff1": 0.9805636864822085, + "nauc_ndcg_at_20_max": 0.39767115483888, + "nauc_ndcg_at_20_std": 0.5925822033717236, + "nauc_ndcg_at_3_diff1": 0.9823805693012515, + "nauc_ndcg_at_3_max": 0.45397611869869464, + "nauc_ndcg_at_3_std": 0.5889900749566161, + "nauc_ndcg_at_5_diff1": 0.9823805693012515, + "nauc_ndcg_at_5_max": 0.45397611869869464, + "nauc_ndcg_at_5_std": 0.5889900749566161, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 0.4143323996265238, + "nauc_precision_at_10_std": 0.7770774976657333, + "nauc_precision_at_1_diff1": 0.9673202614379083, + "nauc_precision_at_1_max": 0.48786181139122375, + "nauc_precision_at_1_std": 0.4282212885154062, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": -0.1713352007469462, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 0.4143323996265135, + "nauc_precision_at_3_std": 0.7770774976657359, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 0.4143323996265238, + "nauc_precision_at_5_std": 0.7770774976657333, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_10_max": 0.4143323996265187, + "nauc_recall_at_10_std": 0.7770774976657333, + "nauc_recall_at_1_diff1": 0.9673202614379083, + "nauc_recall_at_1_max": 0.48786181139122375, + "nauc_recall_at_1_std": 0.4282212885154062, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_20_max": -0.1713352007469626, + "nauc_recall_at_20_std": 1.0, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_3_max": 0.4143323996265187, + "nauc_recall_at_3_std": 0.7770774976657333, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_5_max": 0.4143323996265187, + "nauc_recall_at_5_std": 0.7770774976657333, + "ndcg_at_1": 0.92, + "ndcg_at_10": 0.94524, + "ndcg_at_100": 0.95409, + "ndcg_at_1000": 0.95409, + "ndcg_at_20": 0.95036, + "ndcg_at_3": 0.94524, + "ndcg_at_5": 0.94524, + "precision_at_1": 0.92, + "precision_at_10": 0.096, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.049, + "precision_at_3": 0.32, + "precision_at_5": 0.192, + "recall_at_1": 0.92, + "recall_at_10": 0.96, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.98, + "recall_at_3": 0.96, + "recall_at_5": 0.96 + } + ] + }, + "task_name": "LEMBPasskeyRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LEMBQMSumRetrieval.json b/results/voyageai__voyage-3-lite/no_revision_available/LEMBQMSumRetrieval.json new file mode 100644 index 000000000..786cd095a --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LEMBQMSumRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 24.28458333015442, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.53009, + "map_at_1": 0.36542, + "map_at_10": 0.47139, + "map_at_100": 0.48215, + "map_at_1000": 0.48229, + "map_at_20": 0.47755, + "map_at_3": 0.44161, + "map_at_5": 0.45709, + "mrr_at_1": 0.3654223968565815, + "mrr_at_10": 0.47139088990758976, + "mrr_at_100": 0.48215429765910806, + "mrr_at_1000": 0.48229276973422647, + "mrr_at_20": 0.4775549615191485, + "mrr_at_3": 0.441606636105654, + "mrr_at_5": 0.45709452084697716, + "nauc_map_at_1000_diff1": 0.4984023809974535, + "nauc_map_at_1000_max": -0.07671999854825216, + "nauc_map_at_1000_std": -0.3393618204844076, + "nauc_map_at_100_diff1": 0.4983600991513602, + "nauc_map_at_100_max": -0.07664634883331231, + "nauc_map_at_100_std": -0.3392642687984691, + "nauc_map_at_10_diff1": 0.49482817452295697, + "nauc_map_at_10_max": -0.07881208769714672, + "nauc_map_at_10_std": -0.3415091549277161, + "nauc_map_at_1_diff1": 0.5498833987857136, + "nauc_map_at_1_max": -0.045142607249892525, + "nauc_map_at_1_std": -0.29181693697138533, + "nauc_map_at_20_diff1": 0.4969834357218411, + "nauc_map_at_20_max": -0.07797641570826103, + "nauc_map_at_20_std": -0.34158667079426747, + "nauc_map_at_3_diff1": 0.49667582298809926, + "nauc_map_at_3_max": -0.07414307461715587, + "nauc_map_at_3_std": -0.33167050112644897, + "nauc_map_at_5_diff1": 0.4962020064871691, + "nauc_map_at_5_max": -0.07786215009888524, + "nauc_map_at_5_std": -0.33811695036695316, + "nauc_mrr_at_1000_diff1": 0.4984023809974535, + "nauc_mrr_at_1000_max": -0.07671999854825216, + "nauc_mrr_at_1000_std": -0.3393618204844076, + "nauc_mrr_at_100_diff1": 0.4983600991513602, + "nauc_mrr_at_100_max": -0.07664634883331231, + "nauc_mrr_at_100_std": -0.3392642687984691, + "nauc_mrr_at_10_diff1": 0.49482817452295697, + "nauc_mrr_at_10_max": -0.07881208769714672, + "nauc_mrr_at_10_std": -0.3415091549277161, + "nauc_mrr_at_1_diff1": 0.5498833987857136, + "nauc_mrr_at_1_max": -0.045142607249892525, + "nauc_mrr_at_1_std": -0.29181693697138533, + "nauc_mrr_at_20_diff1": 0.4969834357218411, + "nauc_mrr_at_20_max": -0.07797641570826103, + "nauc_mrr_at_20_std": -0.34158667079426747, + "nauc_mrr_at_3_diff1": 0.49667582298809926, + "nauc_mrr_at_3_max": -0.07414307461715587, + "nauc_mrr_at_3_std": -0.33167050112644897, + "nauc_mrr_at_5_diff1": 0.4962020064871691, + "nauc_mrr_at_5_max": -0.07786215009888524, + "nauc_mrr_at_5_std": -0.33811695036695316, + "nauc_ndcg_at_1000_diff1": 0.49098729663813395, + "nauc_ndcg_at_1000_max": -0.0828747427531432, + "nauc_ndcg_at_1000_std": -0.35020372161460656, + "nauc_ndcg_at_100_diff1": 0.48993756823207757, + "nauc_ndcg_at_100_max": -0.08099297021354714, + "nauc_ndcg_at_100_std": -0.34773564220125175, + "nauc_ndcg_at_10_diff1": 0.47384105109485697, + "nauc_ndcg_at_10_max": -0.0950928397961519, + "nauc_ndcg_at_10_std": -0.36818296399632183, + "nauc_ndcg_at_1_diff1": 0.5498833987857136, + "nauc_ndcg_at_1_max": -0.045142607249892525, + "nauc_ndcg_at_1_std": -0.29181693697138533, + "nauc_ndcg_at_20_diff1": 0.4811497984102747, + "nauc_ndcg_at_20_max": -0.09313973335761352, + "nauc_ndcg_at_20_std": -0.3698463894283154, + "nauc_ndcg_at_3_diff1": 0.4798648243236225, + "nauc_ndcg_at_3_max": -0.08491379403100638, + "nauc_ndcg_at_3_std": -0.345515285930087, + "nauc_ndcg_at_5_diff1": 0.4783313143466035, + "nauc_ndcg_at_5_max": -0.09246760684400858, + "nauc_ndcg_at_5_std": -0.35826740895726555, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.35981233446209215, + "nauc_precision_at_100_max": 0.14762266599302065, + "nauc_precision_at_100_std": -0.04911724453599931, + "nauc_precision_at_10_diff1": 0.3891484005840878, + "nauc_precision_at_10_max": -0.16179440143880655, + "nauc_precision_at_10_std": -0.4814218018560166, + "nauc_precision_at_1_diff1": 0.5498833987857136, + "nauc_precision_at_1_max": -0.045142607249892525, + "nauc_precision_at_1_std": -0.29181693697138533, + "nauc_precision_at_20_diff1": 0.4055336810917817, + "nauc_precision_at_20_max": -0.17484875835973157, + "nauc_precision_at_20_std": -0.5323062845409774, + "nauc_precision_at_3_diff1": 0.42950067751751364, + "nauc_precision_at_3_max": -0.1177216287407601, + "nauc_precision_at_3_std": -0.387413561869622, + "nauc_precision_at_5_diff1": 0.4208307831577087, + "nauc_precision_at_5_max": -0.14128796724029172, + "nauc_precision_at_5_std": -0.42560995858681333, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.3598123344620933, + "nauc_recall_at_100_max": 0.14762266599301604, + "nauc_recall_at_100_std": -0.04911724453601197, + "nauc_recall_at_10_diff1": 0.3891484005840883, + "nauc_recall_at_10_max": -0.1617944014388067, + "nauc_recall_at_10_std": -0.4814218018560162, + "nauc_recall_at_1_diff1": 0.5498833987857136, + "nauc_recall_at_1_max": -0.045142607249892525, + "nauc_recall_at_1_std": -0.29181693697138533, + "nauc_recall_at_20_diff1": 0.4055336810917818, + "nauc_recall_at_20_max": -0.1748487583597322, + "nauc_recall_at_20_std": -0.5323062845409795, + "nauc_recall_at_3_diff1": 0.42950067751751425, + "nauc_recall_at_3_max": -0.11772162874076005, + "nauc_recall_at_3_std": -0.38741356186962217, + "nauc_recall_at_5_diff1": 0.4208307831577084, + "nauc_recall_at_5_max": -0.14128796724029188, + "nauc_recall_at_5_std": -0.42560995858681416, + "ndcg_at_1": 0.36542, + "ndcg_at_10": 0.53009, + "ndcg_at_100": 0.58558, + "ndcg_at_1000": 0.58787, + "ndcg_at_20": 0.55249, + "ndcg_at_3": 0.46753, + "ndcg_at_5": 0.49552, + "precision_at_1": 0.36542, + "precision_at_10": 0.07177, + "precision_at_100": 0.00984, + "precision_at_1000": 0.001, + "precision_at_20": 0.04031, + "precision_at_3": 0.18096, + "precision_at_5": 0.1222, + "recall_at_1": 0.36542, + "recall_at_10": 0.71775, + "recall_at_100": 0.98428, + "recall_at_1000": 1.0, + "recall_at_20": 0.80616, + "recall_at_3": 0.54289, + "recall_at_5": 0.611 + } + ] + }, + "task_name": "LEMBQMSumRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LEMBSummScreenFDRetrieval.json b/results/voyageai__voyage-3-lite/no_revision_available/LEMBSummScreenFDRetrieval.json new file mode 100644 index 000000000..ef5a7700d --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LEMBSummScreenFDRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 22.587260007858276, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "validation": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.96712, + "map_at_1": 0.93452, + "map_at_10": 0.95751, + "map_at_100": 0.95758, + "map_at_1000": 0.95758, + "map_at_20": 0.95751, + "map_at_3": 0.95337, + "map_at_5": 0.95605, + "mrr_at_1": 0.9345238095238095, + "mrr_at_10": 0.9575066137566136, + "mrr_at_100": 0.9575774754346181, + "mrr_at_1000": 0.9575774754346181, + "mrr_at_20": 0.9575066137566136, + "mrr_at_3": 0.953373015873016, + "mrr_at_5": 0.9560515873015873, + "nauc_map_at_1000_diff1": 0.9575407526473582, + "nauc_map_at_1000_max": 0.8219744019530497, + "nauc_map_at_1000_std": 0.044703130036788365, + "nauc_map_at_100_diff1": 0.9575407526473582, + "nauc_map_at_100_max": 0.8219744019530497, + "nauc_map_at_100_std": 0.044703130036788365, + "nauc_map_at_10_diff1": 0.9573932826452728, + "nauc_map_at_10_max": 0.8222712762132698, + "nauc_map_at_10_std": 0.044341704484821685, + "nauc_map_at_1_diff1": 0.9635722751911502, + "nauc_map_at_1_max": 0.7967407191940783, + "nauc_map_at_1_std": 0.03041570424728093, + "nauc_map_at_20_diff1": 0.9573932826452728, + "nauc_map_at_20_max": 0.8222712762132698, + "nauc_map_at_20_std": 0.044341704484821685, + "nauc_map_at_3_diff1": 0.9558085189884304, + "nauc_map_at_3_max": 0.8332368955628943, + "nauc_map_at_3_std": 0.08634675246639753, + "nauc_map_at_5_diff1": 0.9578189972007644, + "nauc_map_at_5_max": 0.8252890103929587, + "nauc_map_at_5_std": 0.041406538464683375, + "nauc_mrr_at_1000_diff1": 0.9575407526473582, + "nauc_mrr_at_1000_max": 0.8219744019530497, + "nauc_mrr_at_1000_std": 0.044703130036788365, + "nauc_mrr_at_100_diff1": 0.9575407526473582, + "nauc_mrr_at_100_max": 0.8219744019530497, + "nauc_mrr_at_100_std": 0.044703130036788365, + "nauc_mrr_at_10_diff1": 0.9573932826452728, + "nauc_mrr_at_10_max": 0.8222712762132698, + "nauc_mrr_at_10_std": 0.044341704484821685, + "nauc_mrr_at_1_diff1": 0.9635722751911502, + "nauc_mrr_at_1_max": 0.7967407191940783, + "nauc_mrr_at_1_std": 0.03041570424728093, + "nauc_mrr_at_20_diff1": 0.9573932826452728, + "nauc_mrr_at_20_max": 0.8222712762132698, + "nauc_mrr_at_20_std": 0.044341704484821685, + "nauc_mrr_at_3_diff1": 0.9558085189884304, + "nauc_mrr_at_3_max": 0.8332368955628943, + "nauc_mrr_at_3_std": 0.08634675246639753, + "nauc_mrr_at_5_diff1": 0.9578189972007644, + "nauc_mrr_at_5_max": 0.8252890103929587, + "nauc_mrr_at_5_std": 0.041406538464683375, + "nauc_ndcg_at_1000_diff1": 0.9567124967273347, + "nauc_ndcg_at_1000_max": 0.8251938277345195, + "nauc_ndcg_at_1000_std": 0.04346391464331009, + "nauc_ndcg_at_100_diff1": 0.9567124967273347, + "nauc_ndcg_at_100_max": 0.8251938277345195, + "nauc_ndcg_at_100_std": 0.04346391464331009, + "nauc_ndcg_at_10_diff1": 0.9552509824692599, + "nauc_ndcg_at_10_max": 0.828110132885467, + "nauc_ndcg_at_10_std": 0.039868783884737004, + "nauc_ndcg_at_1_diff1": 0.9635722751911502, + "nauc_ndcg_at_1_max": 0.7967407191940783, + "nauc_ndcg_at_1_std": 0.03041570424728093, + "nauc_ndcg_at_20_diff1": 0.9552509824692599, + "nauc_ndcg_at_20_max": 0.828110132885467, + "nauc_ndcg_at_20_std": 0.039868783884737004, + "nauc_ndcg_at_3_diff1": 0.9530915678691112, + "nauc_ndcg_at_3_max": 0.851038450223273, + "nauc_ndcg_at_3_std": 0.11985779998178023, + "nauc_ndcg_at_5_diff1": 0.9565861641819805, + "nauc_ndcg_at_5_max": 0.8358121599773046, + "nauc_ndcg_at_5_std": 0.02656936203213249, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 0.8691079080638617, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": -0.1720317259612042, + "nauc_precision_at_1_diff1": 0.9635722751911502, + "nauc_precision_at_1_max": 0.7967407191940783, + "nauc_precision_at_1_std": 0.03041570424728093, + "nauc_precision_at_20_diff1": 0.8691079080638617, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": -0.1720317259612042, + "nauc_precision_at_3_diff1": 0.9400415820086108, + "nauc_precision_at_3_max": 0.9400415820086108, + "nauc_precision_at_3_std": 0.2905139404164876, + "nauc_precision_at_5_diff1": 0.9476431632255661, + "nauc_precision_at_5_max": 0.9182532660027332, + "nauc_precision_at_5_std": -0.13229622346449652, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.8691079080639234, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": -0.17203172596128666, + "nauc_recall_at_1_diff1": 0.9635722751911502, + "nauc_recall_at_1_max": 0.7967407191940783, + "nauc_recall_at_1_std": 0.03041570424728093, + "nauc_recall_at_20_diff1": 0.8691079080639234, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": -0.17203172596128666, + "nauc_recall_at_3_diff1": 0.9400415820086154, + "nauc_recall_at_3_max": 0.9400415820086154, + "nauc_recall_at_3_std": 0.2905139404164915, + "nauc_recall_at_5_diff1": 0.9476431632255614, + "nauc_recall_at_5_max": 0.9182532660027306, + "nauc_recall_at_5_std": -0.13229622346451025, + "ndcg_at_1": 0.93452, + "ndcg_at_10": 0.96712, + "ndcg_at_100": 0.96767, + "ndcg_at_1000": 0.96767, + "ndcg_at_20": 0.96712, + "ndcg_at_3": 0.95855, + "ndcg_at_5": 0.96341, + "precision_at_1": 0.93452, + "precision_at_10": 0.0997, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04985, + "precision_at_3": 0.3244, + "precision_at_5": 0.19702, + "recall_at_1": 0.93452, + "recall_at_10": 0.99702, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.99702, + "recall_at_3": 0.97321, + "recall_at_5": 0.98512 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LEMBWikimQARetrieval.json b/results/voyageai__voyage-3-lite/no_revision_available/LEMBWikimQARetrieval.json new file mode 100644 index 000000000..5060134ba --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LEMBWikimQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 23.969422817230225, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88341, + "map_at_1": 0.80667, + "map_at_10": 0.86057, + "map_at_100": 0.86214, + "map_at_1000": 0.86218, + "map_at_20": 0.86129, + "map_at_3": 0.85167, + "map_at_5": 0.8585, + "mrr_at_1": 0.8066666666666666, + "mrr_at_10": 0.8605727513227514, + "mrr_at_100": 0.8621438621028047, + "mrr_at_1000": 0.8621765418413667, + "mrr_at_20": 0.8612939051689051, + "mrr_at_3": 0.8516666666666668, + "mrr_at_5": 0.8584999999999998, + "nauc_map_at_1000_diff1": 0.8669799492367641, + "nauc_map_at_1000_max": 0.7427065480674543, + "nauc_map_at_1000_std": 0.14863836970821237, + "nauc_map_at_100_diff1": 0.8669805221939116, + "nauc_map_at_100_max": 0.7427380642436305, + "nauc_map_at_100_std": 0.1485586984979546, + "nauc_map_at_10_diff1": 0.8674499056849472, + "nauc_map_at_10_max": 0.7417220388038148, + "nauc_map_at_10_std": 0.14585708900746067, + "nauc_map_at_1_diff1": 0.8807175504312863, + "nauc_map_at_1_max": 0.7531702120347783, + "nauc_map_at_1_std": 0.1634162686002959, + "nauc_map_at_20_diff1": 0.8671861746714196, + "nauc_map_at_20_max": 0.743045568593532, + "nauc_map_at_20_std": 0.14941798817217164, + "nauc_map_at_3_diff1": 0.8606309270653002, + "nauc_map_at_3_max": 0.7392951390286843, + "nauc_map_at_3_std": 0.1479794449100891, + "nauc_map_at_5_diff1": 0.8671146426542917, + "nauc_map_at_5_max": 0.7436244369696322, + "nauc_map_at_5_std": 0.1491710816297423, + "nauc_mrr_at_1000_diff1": 0.8669799492367641, + "nauc_mrr_at_1000_max": 0.7427065480674543, + "nauc_mrr_at_1000_std": 0.14863836970821237, + "nauc_mrr_at_100_diff1": 0.8669805221939116, + "nauc_mrr_at_100_max": 0.7427380642436305, + "nauc_mrr_at_100_std": 0.1485586984979546, + "nauc_mrr_at_10_diff1": 0.8674499056849472, + "nauc_mrr_at_10_max": 0.7417220388038148, + "nauc_mrr_at_10_std": 0.14585708900746067, + "nauc_mrr_at_1_diff1": 0.8807175504312863, + "nauc_mrr_at_1_max": 0.7531702120347783, + "nauc_mrr_at_1_std": 0.1634162686002959, + "nauc_mrr_at_20_diff1": 0.8671861746714196, + "nauc_mrr_at_20_max": 0.743045568593532, + "nauc_mrr_at_20_std": 0.14941798817217164, + "nauc_mrr_at_3_diff1": 0.8606309270653002, + "nauc_mrr_at_3_max": 0.7392951390286843, + "nauc_mrr_at_3_std": 0.1479794449100891, + "nauc_mrr_at_5_diff1": 0.8671146426542917, + "nauc_mrr_at_5_max": 0.7436244369696322, + "nauc_mrr_at_5_std": 0.1491710816297423, + "nauc_ndcg_at_1000_diff1": 0.86629152679548, + "nauc_ndcg_at_1000_max": 0.741362772617396, + "nauc_ndcg_at_1000_std": 0.1453790109253415, + "nauc_ndcg_at_100_diff1": 0.8663060049845566, + "nauc_ndcg_at_100_max": 0.7419822786221557, + "nauc_ndcg_at_100_std": 0.14384516942548528, + "nauc_ndcg_at_10_diff1": 0.8681633268956207, + "nauc_ndcg_at_10_max": 0.7370921288739167, + "nauc_ndcg_at_10_std": 0.1332913020797435, + "nauc_ndcg_at_1_diff1": 0.8807175504312863, + "nauc_ndcg_at_1_max": 0.7531702120347783, + "nauc_ndcg_at_1_std": 0.1634162686002959, + "nauc_ndcg_at_20_diff1": 0.8670882737196772, + "nauc_ndcg_at_20_max": 0.7423320509405916, + "nauc_ndcg_at_20_std": 0.14751153022114472, + "nauc_ndcg_at_3_diff1": 0.8550432040990971, + "nauc_ndcg_at_3_max": 0.7343130326791777, + "nauc_ndcg_at_3_std": 0.14095510812507697, + "nauc_ndcg_at_5_diff1": 0.8670016007791582, + "nauc_ndcg_at_5_max": 0.7428615094009535, + "nauc_ndcg_at_5_std": 0.14254442935574915, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.8692810457516348, + "nauc_precision_at_100_max": 0.8692810457516348, + "nauc_precision_at_100_std": -0.17133520074699457, + "nauc_precision_at_10_diff1": 0.8817860477524349, + "nauc_precision_at_10_max": 0.6970788315326127, + "nauc_precision_at_10_std": 0.02234227024142487, + "nauc_precision_at_1_diff1": 0.8807175504312863, + "nauc_precision_at_1_max": 0.7531702120347783, + "nauc_precision_at_1_std": 0.1634162686002959, + "nauc_precision_at_20_diff1": 0.8733129615482561, + "nauc_precision_at_20_max": 0.7447160682454842, + "nauc_precision_at_20_std": 0.15181224004752733, + "nauc_precision_at_3_diff1": 0.8320161397892494, + "nauc_precision_at_3_max": 0.7130018674136327, + "nauc_precision_at_3_std": 0.11011071095104816, + "nauc_precision_at_5_diff1": 0.8707553196717284, + "nauc_precision_at_5_max": 0.7415352105754589, + "nauc_precision_at_5_std": 0.10199518403852324, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8692810457516407, + "nauc_recall_at_100_max": 0.8692810457516407, + "nauc_recall_at_100_std": -0.17133520074696149, + "nauc_recall_at_10_diff1": 0.8817860477524342, + "nauc_recall_at_10_max": 0.6970788315326123, + "nauc_recall_at_10_std": 0.022342270241428354, + "nauc_recall_at_1_diff1": 0.8807175504312863, + "nauc_recall_at_1_max": 0.7531702120347783, + "nauc_recall_at_1_std": 0.1634162686002959, + "nauc_recall_at_20_diff1": 0.8733129615482579, + "nauc_recall_at_20_max": 0.7447160682454815, + "nauc_recall_at_20_std": 0.15181224004752822, + "nauc_recall_at_3_diff1": 0.832016139789249, + "nauc_recall_at_3_max": 0.7130018674136319, + "nauc_recall_at_3_std": 0.11011071095104794, + "nauc_recall_at_5_diff1": 0.8707553196717288, + "nauc_recall_at_5_max": 0.7415352105754561, + "nauc_recall_at_5_std": 0.10199518403852756, + "ndcg_at_1": 0.80667, + "ndcg_at_10": 0.88341, + "ndcg_at_100": 0.89216, + "ndcg_at_1000": 0.89266, + "ndcg_at_20": 0.88597, + "ndcg_at_3": 0.86583, + "ndcg_at_5": 0.87817, + "precision_at_1": 0.80667, + "precision_at_10": 0.09533, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04817, + "precision_at_3": 0.30222, + "precision_at_5": 0.18733, + "recall_at_1": 0.80667, + "recall_at_10": 0.95333, + "recall_at_100": 0.99667, + "recall_at_1000": 1.0, + "recall_at_20": 0.96333, + "recall_at_3": 0.90667, + "recall_at_5": 0.93667 + } + ] + }, + "task_name": "LEMBWikimQARetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LeCaRDv2.json b/results/voyageai__voyage-3-lite/no_revision_available/LeCaRDv2.json new file mode 100644 index 000000000..ee37fa1e8 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LeCaRDv2.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b78e18688c3d012a33dc3676597c1d1b2243ce1c", + "evaluation_time": 113.95986986160278, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "zho-Hans" + ], + "main_score": 0.67965, + "map_at_1": 0.03884, + "map_at_10": 0.22571, + "map_at_100": 0.52509, + "map_at_1000": 0.54177, + "map_at_20": 0.34851, + "map_at_3": 0.09552, + "map_at_5": 0.14277, + "mrr_at_1": 0.8364779874213837, + "mrr_at_10": 0.8948188080263553, + "mrr_at_100": 0.8948188080263553, + "mrr_at_1000": 0.8948576309163309, + "mrr_at_20": 0.8948188080263553, + "mrr_at_3": 0.8846960167714886, + "mrr_at_5": 0.8922431865828093, + "nauc_map_at_1000_diff1": 0.03077918048309772, + "nauc_map_at_1000_max": 0.4184386133799317, + "nauc_map_at_1000_std": 0.3128868319763606, + "nauc_map_at_100_diff1": 0.02734695506582207, + "nauc_map_at_100_max": 0.40956760274921916, + "nauc_map_at_100_std": 0.3067718940624783, + "nauc_map_at_10_diff1": 0.1415110406689147, + "nauc_map_at_10_max": 0.3543452112258141, + "nauc_map_at_10_std": 0.13280213289969095, + "nauc_map_at_1_diff1": 0.2523470489184731, + "nauc_map_at_1_max": 0.1330906256311314, + "nauc_map_at_1_std": -0.10591015734881322, + "nauc_map_at_20_diff1": 0.09021080199752106, + "nauc_map_at_20_max": 0.3867733907544806, + "nauc_map_at_20_std": 0.19400817692804487, + "nauc_map_at_3_diff1": 0.24080370993731723, + "nauc_map_at_3_max": 0.27518559233365614, + "nauc_map_at_3_std": 0.030669460341604712, + "nauc_map_at_5_diff1": 0.19911090121184902, + "nauc_map_at_5_max": 0.2966668653755003, + "nauc_map_at_5_std": 0.030645599707827854, + "nauc_mrr_at_1000_diff1": 0.22617767432186478, + "nauc_mrr_at_1000_max": 0.6203337376132234, + "nauc_mrr_at_1000_std": 0.44303010293970896, + "nauc_mrr_at_100_diff1": 0.22630336094688003, + "nauc_mrr_at_100_max": 0.6204795779518463, + "nauc_mrr_at_100_std": 0.4432440505206195, + "nauc_mrr_at_10_diff1": 0.22630336094688003, + "nauc_mrr_at_10_max": 0.6204795779518463, + "nauc_mrr_at_10_std": 0.4432440505206195, + "nauc_mrr_at_1_diff1": 0.308402439971479, + "nauc_mrr_at_1_max": 0.6343531905059746, + "nauc_mrr_at_1_std": 0.4545153978992994, + "nauc_mrr_at_20_diff1": 0.22630336094688003, + "nauc_mrr_at_20_max": 0.6204795779518463, + "nauc_mrr_at_20_std": 0.4432440505206195, + "nauc_mrr_at_3_diff1": 0.18542930814659392, + "nauc_mrr_at_3_max": 0.6094849920208859, + "nauc_mrr_at_3_std": 0.4137179279844822, + "nauc_mrr_at_5_diff1": 0.2037440769276679, + "nauc_mrr_at_5_max": 0.619982809901971, + "nauc_mrr_at_5_std": 0.4466663971319747, + "nauc_ndcg_at_1000_diff1": 0.07413205677182733, + "nauc_ndcg_at_1000_max": 0.46324883456965055, + "nauc_ndcg_at_1000_std": 0.36866479055942464, + "nauc_ndcg_at_100_diff1": 0.03822772826941456, + "nauc_ndcg_at_100_max": 0.4030577843331345, + "nauc_ndcg_at_100_std": 0.3706514862097985, + "nauc_ndcg_at_10_diff1": 0.0490740894641729, + "nauc_ndcg_at_10_max": 0.4491144969020808, + "nauc_ndcg_at_10_std": 0.32983008448656737, + "nauc_ndcg_at_1_diff1": 0.308402439971479, + "nauc_ndcg_at_1_max": 0.6343531905059746, + "nauc_ndcg_at_1_std": 0.4545153978992994, + "nauc_ndcg_at_20_diff1": 0.05443011537477711, + "nauc_ndcg_at_20_max": 0.43162892323306296, + "nauc_ndcg_at_20_std": 0.3031022378145758, + "nauc_ndcg_at_3_diff1": 0.13109736151998924, + "nauc_ndcg_at_3_max": 0.5618326219188247, + "nauc_ndcg_at_3_std": 0.4136226056887153, + "nauc_ndcg_at_5_diff1": 0.051742688672249765, + "nauc_ndcg_at_5_max": 0.4884251566689633, + "nauc_ndcg_at_5_std": 0.3392417400026412, + "nauc_precision_at_1000_diff1": -0.11094852397846527, + "nauc_precision_at_1000_max": 0.33904140912649067, + "nauc_precision_at_1000_std": 0.44359873455117943, + "nauc_precision_at_100_diff1": -0.10140089571616995, + "nauc_precision_at_100_max": 0.3207472454690883, + "nauc_precision_at_100_std": 0.4503687517286694, + "nauc_precision_at_10_diff1": -0.0011472368356578624, + "nauc_precision_at_10_max": 0.3842869744816796, + "nauc_precision_at_10_std": 0.3111343700808287, + "nauc_precision_at_1_diff1": 0.308402439971479, + "nauc_precision_at_1_max": 0.6343531905059746, + "nauc_precision_at_1_std": 0.4545153978992994, + "nauc_precision_at_20_diff1": -0.001190009393790274, + "nauc_precision_at_20_max": 0.37929693712793844, + "nauc_precision_at_20_std": 0.3044033924249466, + "nauc_precision_at_3_diff1": 0.08371508673246111, + "nauc_precision_at_3_max": 0.5428351378236234, + "nauc_precision_at_3_std": 0.40834112167158887, + "nauc_precision_at_5_diff1": -0.01772849658614705, + "nauc_precision_at_5_max": 0.4336278187751823, + "nauc_precision_at_5_std": 0.30812392933127886, + "nauc_recall_at_1000_diff1": 0.6400334564005784, + "nauc_recall_at_1000_max": 0.986440388879424, + "nauc_recall_at_1000_std": 0.9711724802948536, + "nauc_recall_at_100_diff1": -0.053067579255731585, + "nauc_recall_at_100_max": 0.20178990556183235, + "nauc_recall_at_100_std": 0.3751428371270488, + "nauc_recall_at_10_diff1": 0.11673918112870137, + "nauc_recall_at_10_max": 0.2115256678688023, + "nauc_recall_at_10_std": 0.036183686405075854, + "nauc_recall_at_1_diff1": 0.2523470489184731, + "nauc_recall_at_1_max": 0.1330906256311314, + "nauc_recall_at_1_std": -0.10591015734881322, + "nauc_recall_at_20_diff1": 0.10401368594796634, + "nauc_recall_at_20_max": 0.27241455628532596, + "nauc_recall_at_20_std": 0.1156335367348927, + "nauc_recall_at_3_diff1": 0.19499936979829238, + "nauc_recall_at_3_max": 0.2296891076584978, + "nauc_recall_at_3_std": -0.017645845309342382, + "nauc_recall_at_5_diff1": 0.14669357918638445, + "nauc_recall_at_5_max": 0.19729911931668856, + "nauc_recall_at_5_std": -0.04055102125506616, + "ndcg_at_1": 0.83648, + "ndcg_at_10": 0.67965, + "ndcg_at_100": 0.75719, + "ndcg_at_1000": 0.81186, + "ndcg_at_20": 0.62095, + "ndcg_at_3": 0.77351, + "ndcg_at_5": 0.74559, + "precision_at_1": 0.83648, + "precision_at_10": 0.6283, + "precision_at_100": 0.21201, + "precision_at_1000": 0.02448, + "precision_at_20": 0.53679, + "precision_at_3": 0.75472, + "precision_at_5": 0.71824, + "recall_at_1": 0.03884, + "recall_at_10": 0.26715, + "recall_at_100": 0.86032, + "recall_at_1000": 0.99783, + "recall_at_20": 0.44335, + "recall_at_3": 0.10079, + "recall_at_5": 0.15923 + } + ] + }, + "task_name": "LeCaRDv2" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LegalBenchConsumerContractsQA.json b/results/voyageai__voyage-3-lite/no_revision_available/LegalBenchConsumerContractsQA.json new file mode 100644 index 000000000..360174cd0 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LegalBenchConsumerContractsQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b23590301ec94e8087e2850b21d43d4956b1cca9", + "evaluation_time": 3.11063551902771, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.8322, + "map_at_1": 0.70455, + "map_at_10": 0.78993, + "map_at_100": 0.79168, + "map_at_1000": 0.79168, + "map_at_20": 0.79116, + "map_at_3": 0.77146, + "map_at_5": 0.78346, + "mrr_at_1": 0.7045454545454546, + "mrr_at_10": 0.7899310565977231, + "mrr_at_100": 0.7916839268832538, + "mrr_at_1000": 0.7916839268832538, + "mrr_at_20": 0.7911559305387289, + "mrr_at_3": 0.7714646464646469, + "mrr_at_5": 0.7834595959595961, + "nauc_map_at_1000_diff1": 0.5377349355873945, + "nauc_map_at_1000_max": 0.44715574371606215, + "nauc_map_at_1000_std": 0.1215964852880569, + "nauc_map_at_100_diff1": 0.5377349355873945, + "nauc_map_at_100_max": 0.44715574371606215, + "nauc_map_at_100_std": 0.1215964852880569, + "nauc_map_at_10_diff1": 0.535353583578055, + "nauc_map_at_10_max": 0.4472911036789225, + "nauc_map_at_10_std": 0.11933521935286054, + "nauc_map_at_1_diff1": 0.573517889605697, + "nauc_map_at_1_max": 0.45464215876553826, + "nauc_map_at_1_std": 0.11855253113192336, + "nauc_map_at_20_diff1": 0.536101711005393, + "nauc_map_at_20_max": 0.4467294014247577, + "nauc_map_at_20_std": 0.1217529803788588, + "nauc_map_at_3_diff1": 0.5319821556133733, + "nauc_map_at_3_max": 0.44510812700859165, + "nauc_map_at_3_std": 0.11568316506108327, + "nauc_map_at_5_diff1": 0.5305296005403388, + "nauc_map_at_5_max": 0.44710553180312146, + "nauc_map_at_5_std": 0.12725713911261155, + "nauc_mrr_at_1000_diff1": 0.5377349355873945, + "nauc_mrr_at_1000_max": 0.45323940716454864, + "nauc_mrr_at_1000_std": 0.09630583302150554, + "nauc_mrr_at_100_diff1": 0.5377349355873945, + "nauc_mrr_at_100_max": 0.45323940716454864, + "nauc_mrr_at_100_std": 0.09630583302150554, + "nauc_mrr_at_10_diff1": 0.535353583578055, + "nauc_mrr_at_10_max": 0.4533178374074208, + "nauc_mrr_at_10_std": 0.09428123201099545, + "nauc_mrr_at_1_diff1": 0.573517889605697, + "nauc_mrr_at_1_max": 0.46346499270235725, + "nauc_mrr_at_1_std": 0.08187475843936387, + "nauc_mrr_at_20_diff1": 0.536101711005393, + "nauc_mrr_at_20_max": 0.45279580373666206, + "nauc_mrr_at_20_std": 0.0965340851065462, + "nauc_mrr_at_3_diff1": 0.5319821556133733, + "nauc_mrr_at_3_max": 0.4506254920087113, + "nauc_mrr_at_3_std": 0.09274669580580616, + "nauc_mrr_at_5_diff1": 0.5305296005403388, + "nauc_mrr_at_5_max": 0.4529310054502065, + "nauc_mrr_at_5_std": 0.1030398184892888, + "nauc_ndcg_at_1000_diff1": 0.5332855600607193, + "nauc_ndcg_at_1000_max": 0.4473432416030445, + "nauc_ndcg_at_1000_std": 0.12364222129388426, + "nauc_ndcg_at_100_diff1": 0.5332855600607193, + "nauc_ndcg_at_100_max": 0.4473432416030445, + "nauc_ndcg_at_100_std": 0.12364222129388426, + "nauc_ndcg_at_10_diff1": 0.52115868445175, + "nauc_ndcg_at_10_max": 0.4475811356665206, + "nauc_ndcg_at_10_std": 0.11745607578466376, + "nauc_ndcg_at_1_diff1": 0.573517889605697, + "nauc_ndcg_at_1_max": 0.45464215876553826, + "nauc_ndcg_at_1_std": 0.11855253113192336, + "nauc_ndcg_at_20_diff1": 0.5245478067466348, + "nauc_ndcg_at_20_max": 0.44502536244688407, + "nauc_ndcg_at_20_std": 0.12656139576146275, + "nauc_ndcg_at_3_diff1": 0.5147588004688498, + "nauc_ndcg_at_3_max": 0.4450536033857644, + "nauc_ndcg_at_3_std": 0.11559729536604925, + "nauc_ndcg_at_5_diff1": 0.5092924678944567, + "nauc_ndcg_at_5_max": 0.44857125021371724, + "nauc_ndcg_at_5_std": 0.139332324389732, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 0.3206515456371885, + "nauc_precision_at_10_max": 0.45115927080179874, + "nauc_precision_at_10_std": 0.0675464497478786, + "nauc_precision_at_1_diff1": 0.573517889605697, + "nauc_precision_at_1_max": 0.45464215876553826, + "nauc_precision_at_1_std": 0.11855253113192336, + "nauc_precision_at_20_diff1": 0.23776895575025858, + "nauc_precision_at_20_max": 0.37202384604445726, + "nauc_precision_at_20_std": 0.28186159075452233, + "nauc_precision_at_3_diff1": 0.43914184007052987, + "nauc_precision_at_3_max": 0.4466036128672141, + "nauc_precision_at_3_std": 0.11572695180584049, + "nauc_precision_at_5_diff1": 0.3691502723835559, + "nauc_precision_at_5_max": 0.461145487775082, + "nauc_precision_at_5_std": 0.22571451623121336, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.32065154563718706, + "nauc_recall_at_10_max": 0.4511592708017948, + "nauc_recall_at_10_std": 0.06754644974787737, + "nauc_recall_at_1_diff1": 0.573517889605697, + "nauc_recall_at_1_max": 0.45464215876553826, + "nauc_recall_at_1_std": 0.11855253113192336, + "nauc_recall_at_20_diff1": 0.2377689557502527, + "nauc_recall_at_20_max": 0.37202384604445937, + "nauc_recall_at_20_std": 0.28186159075451234, + "nauc_recall_at_3_diff1": 0.4391418400705295, + "nauc_recall_at_3_max": 0.44660361286721467, + "nauc_recall_at_3_std": 0.11572695180584038, + "nauc_recall_at_5_diff1": 0.3691502723835541, + "nauc_recall_at_5_max": 0.46114548777508074, + "nauc_recall_at_5_std": 0.22571451623121558, + "ndcg_at_1": 0.70455, + "ndcg_at_10": 0.8322, + "ndcg_at_100": 0.84007, + "ndcg_at_1000": 0.84007, + "ndcg_at_20": 0.83666, + "ndcg_at_3": 0.7944, + "ndcg_at_5": 0.81613, + "precision_at_1": 0.70455, + "precision_at_10": 0.09646, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04912, + "precision_at_3": 0.28704, + "precision_at_5": 0.18283, + "recall_at_1": 0.70455, + "recall_at_10": 0.96465, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.98232, + "recall_at_3": 0.86111, + "recall_at_5": 0.91414 + } + ] + }, + "task_name": "LegalBenchConsumerContractsQA" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LegalBenchCorporateLobbying.json b/results/voyageai__voyage-3-lite/no_revision_available/LegalBenchCorporateLobbying.json new file mode 100644 index 000000000..1f359db73 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LegalBenchCorporateLobbying.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f69691c650464e62546d7f2a4536f8f87c891e38", + "evaluation_time": 3.7167882919311523, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94528, + "map_at_1": 0.88529, + "map_at_10": 0.93065, + "map_at_100": 0.93091, + "map_at_1000": 0.93094, + "map_at_20": 0.93085, + "map_at_3": 0.92941, + "map_at_5": 0.92941, + "mrr_at_1": 0.8852941176470588, + "mrr_at_10": 0.9306489262371612, + "mrr_at_100": 0.930910364145658, + "mrr_at_1000": 0.9309390261386827, + "mrr_at_20": 0.9308450046685339, + "mrr_at_3": 0.9294117647058824, + "mrr_at_5": 0.9294117647058824, + "nauc_map_at_1000_diff1": 0.8857601607451715, + "nauc_map_at_1000_max": 0.5689351865966871, + "nauc_map_at_1000_std": 0.020996633639007452, + "nauc_map_at_100_diff1": 0.8856901386498335, + "nauc_map_at_100_max": 0.5688842040768617, + "nauc_map_at_100_std": 0.020682818071194587, + "nauc_map_at_10_diff1": 0.8857523373555236, + "nauc_map_at_10_max": 0.5701574745183051, + "nauc_map_at_10_std": 0.022586071123425474, + "nauc_map_at_1_diff1": 0.8847184007402179, + "nauc_map_at_1_max": 0.5148287029146069, + "nauc_map_at_1_std": 0.015376561396674293, + "nauc_map_at_20_diff1": 0.8857996974346373, + "nauc_map_at_20_max": 0.5692974013474483, + "nauc_map_at_20_std": 0.021621431757907567, + "nauc_map_at_3_diff1": 0.8877823354631745, + "nauc_map_at_3_max": 0.5777950882175064, + "nauc_map_at_3_std": 0.018057637190711916, + "nauc_map_at_5_diff1": 0.8877823354631745, + "nauc_map_at_5_max": 0.5777950882175064, + "nauc_map_at_5_std": 0.018057637190711916, + "nauc_mrr_at_1000_diff1": 0.8857601607451715, + "nauc_mrr_at_1000_max": 0.5689351865966871, + "nauc_mrr_at_1000_std": 0.020996633639007452, + "nauc_mrr_at_100_diff1": 0.8856901386498335, + "nauc_mrr_at_100_max": 0.5688842040768617, + "nauc_mrr_at_100_std": 0.020682818071194587, + "nauc_mrr_at_10_diff1": 0.8857523373555236, + "nauc_mrr_at_10_max": 0.5701574745183051, + "nauc_mrr_at_10_std": 0.022586071123425474, + "nauc_mrr_at_1_diff1": 0.8847184007402179, + "nauc_mrr_at_1_max": 0.5148287029146069, + "nauc_mrr_at_1_std": 0.015376561396674293, + "nauc_mrr_at_20_diff1": 0.8857996974346373, + "nauc_mrr_at_20_max": 0.5692974013474483, + "nauc_mrr_at_20_std": 0.021621431757907567, + "nauc_mrr_at_3_diff1": 0.8877823354631745, + "nauc_mrr_at_3_max": 0.5777950882175064, + "nauc_mrr_at_3_std": 0.018057637190711916, + "nauc_mrr_at_5_diff1": 0.8877823354631745, + "nauc_mrr_at_5_max": 0.5777950882175064, + "nauc_mrr_at_5_std": 0.018057637190711916, + "nauc_ndcg_at_1000_diff1": 0.8853618938494467, + "nauc_ndcg_at_1000_max": 0.5757217867050933, + "nauc_ndcg_at_1000_std": 0.02082171830505301, + "nauc_ndcg_at_100_diff1": 0.8836390314795101, + "nauc_ndcg_at_100_max": 0.575203004277269, + "nauc_ndcg_at_100_std": 0.014201940281359932, + "nauc_ndcg_at_10_diff1": 0.8845907175818843, + "nauc_ndcg_at_10_max": 0.5833943344398418, + "nauc_ndcg_at_10_std": 0.028601832137360197, + "nauc_ndcg_at_1_diff1": 0.8847184007402179, + "nauc_ndcg_at_1_max": 0.5148287029146069, + "nauc_ndcg_at_1_std": 0.015376561396674293, + "nauc_ndcg_at_20_diff1": 0.8848020133132235, + "nauc_ndcg_at_20_max": 0.5794486821478002, + "nauc_ndcg_at_20_std": 0.02405460178581354, + "nauc_ndcg_at_3_diff1": 0.8904973758791739, + "nauc_ndcg_at_3_max": 0.6047162529165894, + "nauc_ndcg_at_3_std": 0.013825183281769543, + "nauc_ndcg_at_5_diff1": 0.8904973758791739, + "nauc_ndcg_at_5_max": 0.6047162529165894, + "nauc_ndcg_at_5_std": 0.013825183281769543, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7770774976657122, + "nauc_precision_at_100_max": 0.561391223155907, + "nauc_precision_at_100_std": -0.3699813258637037, + "nauc_precision_at_10_diff1": 0.8558590102707839, + "nauc_precision_at_10_max": 0.7480158730158858, + "nauc_precision_at_10_std": 0.15452847805789968, + "nauc_precision_at_1_diff1": 0.8847184007402179, + "nauc_precision_at_1_max": 0.5148287029146069, + "nauc_precision_at_1_std": 0.015376561396674293, + "nauc_precision_at_20_diff1": 0.851384998443819, + "nauc_precision_at_20_max": 0.7075941487706214, + "nauc_precision_at_20_std": 0.08667911609089299, + "nauc_precision_at_3_diff1": 0.9176337201547229, + "nauc_precision_at_3_max": 0.8560090702947908, + "nauc_precision_at_3_std": -0.03254635187408526, + "nauc_precision_at_5_diff1": 0.9176337201547251, + "nauc_precision_at_5_max": 0.8560090702947828, + "nauc_precision_at_5_std": -0.03254635187409196, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7770774976657118, + "nauc_recall_at_100_max": 0.5613912231559266, + "nauc_recall_at_100_std": -0.36998132586367716, + "nauc_recall_at_10_diff1": 0.855859010270774, + "nauc_recall_at_10_max": 0.7480158730158691, + "nauc_recall_at_10_std": 0.15452847805788197, + "nauc_recall_at_1_diff1": 0.8847184007402179, + "nauc_recall_at_1_max": 0.5148287029146069, + "nauc_recall_at_1_std": 0.015376561396674293, + "nauc_recall_at_20_diff1": 0.8513849984438363, + "nauc_recall_at_20_max": 0.7075941487706155, + "nauc_recall_at_20_std": 0.08667911609087513, + "nauc_recall_at_3_diff1": 0.9176337201547229, + "nauc_recall_at_3_max": 0.8560090702947845, + "nauc_recall_at_3_std": -0.03254635187408526, + "nauc_recall_at_5_diff1": 0.9176337201547229, + "nauc_recall_at_5_max": 0.8560090702947845, + "nauc_recall_at_5_std": -0.03254635187408526, + "ndcg_at_1": 0.88529, + "ndcg_at_10": 0.94528, + "ndcg_at_100": 0.94655, + "ndcg_at_1000": 0.94731, + "ndcg_at_20": 0.94601, + "ndcg_at_3": 0.94237, + "ndcg_at_5": 0.94237, + "precision_at_1": 0.88529, + "precision_at_10": 0.09882, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04956, + "precision_at_3": 0.32647, + "precision_at_5": 0.19588, + "recall_at_1": 0.88529, + "recall_at_10": 0.98824, + "recall_at_100": 0.99412, + "recall_at_1000": 1.0, + "recall_at_20": 0.99118, + "recall_at_3": 0.97941, + "recall_at_5": 0.97941 + } + ] + }, + "task_name": "LegalBenchCorporateLobbying" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LegalQuAD.json b/results/voyageai__voyage-3-lite/no_revision_available/LegalQuAD.json new file mode 100644 index 000000000..dc767aa5d --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LegalQuAD.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e", + "evaluation_time": 10.232067346572876, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "deu-Latn" + ], + "main_score": 0.61963, + "map_at_1": 0.465, + "map_at_10": 0.57, + "map_at_100": 0.57896, + "map_at_1000": 0.57913, + "map_at_20": 0.576, + "map_at_3": 0.54667, + "map_at_5": 0.56242, + "mrr_at_1": 0.465, + "mrr_at_10": 0.5700039682539683, + "mrr_at_100": 0.5789572841499331, + "mrr_at_1000": 0.5791262205921133, + "mrr_at_20": 0.5760035540439952, + "mrr_at_3": 0.5466666666666666, + "mrr_at_5": 0.5624166666666667, + "nauc_map_at_1000_diff1": 0.6261380262356616, + "nauc_map_at_1000_max": 0.6779872167035177, + "nauc_map_at_1000_std": 0.3592964456303077, + "nauc_map_at_100_diff1": 0.6258659014170076, + "nauc_map_at_100_max": 0.6779614888055631, + "nauc_map_at_100_std": 0.35952825707978114, + "nauc_map_at_10_diff1": 0.6239038160480936, + "nauc_map_at_10_max": 0.6805273313182092, + "nauc_map_at_10_std": 0.36514650774671636, + "nauc_map_at_1_diff1": 0.6940128076029511, + "nauc_map_at_1_max": 0.6636260127286249, + "nauc_map_at_1_std": 0.2669654668588463, + "nauc_map_at_20_diff1": 0.62629461609084, + "nauc_map_at_20_max": 0.6785078246677351, + "nauc_map_at_20_std": 0.36129935266487345, + "nauc_map_at_3_diff1": 0.6141543101640967, + "nauc_map_at_3_max": 0.6820205740532501, + "nauc_map_at_3_std": 0.3499876117326419, + "nauc_map_at_5_diff1": 0.623945341704446, + "nauc_map_at_5_max": 0.6800786786825805, + "nauc_map_at_5_std": 0.36978809196717216, + "nauc_mrr_at_1000_diff1": 0.6261380262356616, + "nauc_mrr_at_1000_max": 0.6779872167035177, + "nauc_mrr_at_1000_std": 0.3592964456303077, + "nauc_mrr_at_100_diff1": 0.6258659014170076, + "nauc_mrr_at_100_max": 0.6779614888055631, + "nauc_mrr_at_100_std": 0.35952825707978114, + "nauc_mrr_at_10_diff1": 0.6239038160480936, + "nauc_mrr_at_10_max": 0.6805273313182092, + "nauc_mrr_at_10_std": 0.36514650774671636, + "nauc_mrr_at_1_diff1": 0.6940128076029511, + "nauc_mrr_at_1_max": 0.6636260127286249, + "nauc_mrr_at_1_std": 0.2669654668588463, + "nauc_mrr_at_20_diff1": 0.62629461609084, + "nauc_mrr_at_20_max": 0.6785078246677351, + "nauc_mrr_at_20_std": 0.36129935266487345, + "nauc_mrr_at_3_diff1": 0.6141543101640967, + "nauc_mrr_at_3_max": 0.6820205740532501, + "nauc_mrr_at_3_std": 0.3499876117326419, + "nauc_mrr_at_5_diff1": 0.623945341704446, + "nauc_mrr_at_5_max": 0.6800786786825805, + "nauc_mrr_at_5_std": 0.36978809196717216, + "nauc_ndcg_at_1000_diff1": 0.615276418562554, + "nauc_ndcg_at_1000_max": 0.6775817814398449, + "nauc_ndcg_at_1000_std": 0.37631930296482335, + "nauc_ndcg_at_100_diff1": 0.6077961266773739, + "nauc_ndcg_at_100_max": 0.6763204669568083, + "nauc_ndcg_at_100_std": 0.38146220649795715, + "nauc_ndcg_at_10_diff1": 0.6023241209533817, + "nauc_ndcg_at_10_max": 0.6852887790129808, + "nauc_ndcg_at_10_std": 0.4094119830147255, + "nauc_ndcg_at_1_diff1": 0.6940128076029511, + "nauc_ndcg_at_1_max": 0.6636260127286249, + "nauc_ndcg_at_1_std": 0.2669654668588463, + "nauc_ndcg_at_20_diff1": 0.6096781348017634, + "nauc_ndcg_at_20_max": 0.6778579933652034, + "nauc_ndcg_at_20_std": 0.39533170622459707, + "nauc_ndcg_at_3_diff1": 0.5864823191521837, + "nauc_ndcg_at_3_max": 0.6862507108433773, + "nauc_ndcg_at_3_std": 0.37744582604514687, + "nauc_ndcg_at_5_diff1": 0.6036180111825403, + "nauc_ndcg_at_5_max": 0.683402082331299, + "nauc_ndcg_at_5_std": 0.41834716102595004, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.062278244631194704, + "nauc_precision_at_100_max": 0.5761904761904957, + "nauc_precision_at_100_std": 0.7395891690009454, + "nauc_precision_at_10_diff1": 0.5105278141547571, + "nauc_precision_at_10_max": 0.7069635028184255, + "nauc_precision_at_10_std": 0.6100324545920401, + "nauc_precision_at_1_diff1": 0.6940128076029511, + "nauc_precision_at_1_max": 0.6636260127286249, + "nauc_precision_at_1_std": 0.2669654668588463, + "nauc_precision_at_20_diff1": 0.5189100305379376, + "nauc_precision_at_20_max": 0.6629081512802432, + "nauc_precision_at_20_std": 0.5932246048525113, + "nauc_precision_at_3_diff1": 0.4966593325112628, + "nauc_precision_at_3_max": 0.6992270928536308, + "nauc_precision_at_3_std": 0.4661137841824028, + "nauc_precision_at_5_diff1": 0.5320881000949429, + "nauc_precision_at_5_max": 0.6947329830135649, + "nauc_precision_at_5_std": 0.6055988493969676, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.06227824463118748, + "nauc_recall_at_100_max": 0.576190476190479, + "nauc_recall_at_100_std": 0.7395891690009352, + "nauc_recall_at_10_diff1": 0.5105278141547567, + "nauc_recall_at_10_max": 0.7069635028184252, + "nauc_recall_at_10_std": 0.6100324545920405, + "nauc_recall_at_1_diff1": 0.6940128076029511, + "nauc_recall_at_1_max": 0.6636260127286249, + "nauc_recall_at_1_std": 0.2669654668588463, + "nauc_recall_at_20_diff1": 0.5189100305379379, + "nauc_recall_at_20_max": 0.6629081512802435, + "nauc_recall_at_20_std": 0.5932246048525115, + "nauc_recall_at_3_diff1": 0.49665933251126326, + "nauc_recall_at_3_max": 0.6992270928536314, + "nauc_recall_at_3_std": 0.46611378418240246, + "nauc_recall_at_5_diff1": 0.5320881000949435, + "nauc_recall_at_5_max": 0.6947329830135648, + "nauc_recall_at_5_std": 0.6055988493969676, + "ndcg_at_1": 0.465, + "ndcg_at_10": 0.61963, + "ndcg_at_100": 0.66236, + "ndcg_at_1000": 0.66581, + "ndcg_at_20": 0.6404, + "ndcg_at_3": 0.57202, + "ndcg_at_5": 0.60063, + "precision_at_1": 0.465, + "precision_at_10": 0.0775, + "precision_at_100": 0.00975, + "precision_at_1000": 0.001, + "precision_at_20": 0.04275, + "precision_at_3": 0.215, + "precision_at_5": 0.143, + "recall_at_1": 0.465, + "recall_at_10": 0.775, + "recall_at_100": 0.975, + "recall_at_1000": 1.0, + "recall_at_20": 0.855, + "recall_at_3": 0.645, + "recall_at_5": 0.715 + } + ] + }, + "task_name": "LegalQuAD" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/LegalSummarization.json b/results/voyageai__voyage-3-lite/no_revision_available/LegalSummarization.json new file mode 100644 index 000000000..cbf954875 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/LegalSummarization.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3bb1a05c66872889662af04c5691c14489cebd72", + "evaluation_time": 2.6194822788238525, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.61418, + "map_at_1": 0.44253, + "map_at_10": 0.55097, + "map_at_100": 0.56802, + "map_at_1000": 0.56853, + "map_at_20": 0.56265, + "map_at_3": 0.512, + "map_at_5": 0.53482, + "mrr_at_1": 0.5105633802816901, + "mrr_at_10": 0.6064009054325953, + "mrr_at_100": 0.6138025929041718, + "mrr_at_1000": 0.614011299316746, + "mrr_at_20": 0.6122291911510775, + "mrr_at_3": 0.5757042253521126, + "mrr_at_5": 0.5957746478873239, + "nauc_map_at_1000_diff1": 0.6692694242457398, + "nauc_map_at_1000_max": 0.5770923346387064, + "nauc_map_at_1000_std": 0.07942781638896461, + "nauc_map_at_100_diff1": 0.6692351312547501, + "nauc_map_at_100_max": 0.5772283267127903, + "nauc_map_at_100_std": 0.079742350775488, + "nauc_map_at_10_diff1": 0.6712546394053011, + "nauc_map_at_10_max": 0.5678306567677543, + "nauc_map_at_10_std": 0.06386335412742672, + "nauc_map_at_1_diff1": 0.6953114123108786, + "nauc_map_at_1_max": 0.572922171709509, + "nauc_map_at_1_std": 0.037438363048026425, + "nauc_map_at_20_diff1": 0.670065683276195, + "nauc_map_at_20_max": 0.5757222176630755, + "nauc_map_at_20_std": 0.07398363101850736, + "nauc_map_at_3_diff1": 0.6713026091596059, + "nauc_map_at_3_max": 0.5534769245653782, + "nauc_map_at_3_std": 0.04643285265547695, + "nauc_map_at_5_diff1": 0.6701246085504504, + "nauc_map_at_5_max": 0.5587808148009481, + "nauc_map_at_5_std": 0.04883557512996957, + "nauc_mrr_at_1000_diff1": 0.6430097453992788, + "nauc_mrr_at_1000_max": 0.5646550555855211, + "nauc_mrr_at_1000_std": 0.10357223030724325, + "nauc_mrr_at_100_diff1": 0.6430103298214014, + "nauc_mrr_at_100_max": 0.5646844220107706, + "nauc_mrr_at_100_std": 0.10377188471239546, + "nauc_mrr_at_10_diff1": 0.6426195290832389, + "nauc_mrr_at_10_max": 0.5636045279513954, + "nauc_mrr_at_10_std": 0.10585192975522718, + "nauc_mrr_at_1_diff1": 0.6627794537013902, + "nauc_mrr_at_1_max": 0.5637272971781279, + "nauc_mrr_at_1_std": 0.07435969780461078, + "nauc_mrr_at_20_diff1": 0.6422317528102812, + "nauc_mrr_at_20_max": 0.5656329234480061, + "nauc_mrr_at_20_std": 0.10626045287658983, + "nauc_mrr_at_3_diff1": 0.641415733028211, + "nauc_mrr_at_3_max": 0.5615733156009203, + "nauc_mrr_at_3_std": 0.09376772474262092, + "nauc_mrr_at_5_diff1": 0.6436583786845806, + "nauc_mrr_at_5_max": 0.5594124485764416, + "nauc_mrr_at_5_std": 0.0965803562947019, + "nauc_ndcg_at_1000_diff1": 0.6585228202766013, + "nauc_ndcg_at_1000_max": 0.5817471518832449, + "nauc_ndcg_at_1000_std": 0.10984632203972648, + "nauc_ndcg_at_100_diff1": 0.6567892471056258, + "nauc_ndcg_at_100_max": 0.5834294982875229, + "nauc_ndcg_at_100_std": 0.11653286024069746, + "nauc_ndcg_at_10_diff1": 0.659700660433683, + "nauc_ndcg_at_10_max": 0.5664965946847962, + "nauc_ndcg_at_10_std": 0.08925347052423344, + "nauc_ndcg_at_1_diff1": 0.6627794537013902, + "nauc_ndcg_at_1_max": 0.5637272971781279, + "nauc_ndcg_at_1_std": 0.07435969780461078, + "nauc_ndcg_at_20_diff1": 0.6566636062447283, + "nauc_ndcg_at_20_max": 0.5863507980948948, + "nauc_ndcg_at_20_std": 0.113647272425289, + "nauc_ndcg_at_3_diff1": 0.6499991425402162, + "nauc_ndcg_at_3_max": 0.5588313622805454, + "nauc_ndcg_at_3_std": 0.07880978715657123, + "nauc_ndcg_at_5_diff1": 0.6548669767739419, + "nauc_ndcg_at_5_max": 0.5584577462324612, + "nauc_ndcg_at_5_std": 0.07134322603138483, + "nauc_precision_at_1000_diff1": -0.23977390459079959, + "nauc_precision_at_1000_max": -0.0469105782141164, + "nauc_precision_at_1000_std": 0.2806736372405332, + "nauc_precision_at_100_diff1": -0.19921541428088294, + "nauc_precision_at_100_max": 0.00560549819772252, + "nauc_precision_at_100_std": 0.31487107019460364, + "nauc_precision_at_10_diff1": 0.0962389348547682, + "nauc_precision_at_10_max": 0.22703638593593273, + "nauc_precision_at_10_std": 0.28202408089974923, + "nauc_precision_at_1_diff1": 0.6627794537013902, + "nauc_precision_at_1_max": 0.5637272971781279, + "nauc_precision_at_1_std": 0.07435969780461078, + "nauc_precision_at_20_diff1": -0.0707045355094972, + "nauc_precision_at_20_max": 0.13646429968477694, + "nauc_precision_at_20_std": 0.33886465469897586, + "nauc_precision_at_3_diff1": 0.38798616960982907, + "nauc_precision_at_3_max": 0.3811212852924146, + "nauc_precision_at_3_std": 0.19534549447712737, + "nauc_precision_at_5_diff1": 0.24276079938193298, + "nauc_precision_at_5_max": 0.2916015572973197, + "nauc_precision_at_5_std": 0.20941123233613457, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5303661335469185, + "nauc_recall_at_100_max": 0.574815377254243, + "nauc_recall_at_100_std": 0.33238422567905224, + "nauc_recall_at_10_diff1": 0.5905931318331495, + "nauc_recall_at_10_max": 0.5080339712113687, + "nauc_recall_at_10_std": 0.1152725005198199, + "nauc_recall_at_1_diff1": 0.6953114123108786, + "nauc_recall_at_1_max": 0.572922171709509, + "nauc_recall_at_1_std": 0.037438363048026425, + "nauc_recall_at_20_diff1": 0.5437776782316784, + "nauc_recall_at_20_max": 0.5925064522021677, + "nauc_recall_at_20_std": 0.2525542953416983, + "nauc_recall_at_3_diff1": 0.6237200184903023, + "nauc_recall_at_3_max": 0.510839983650413, + "nauc_recall_at_3_std": 0.042107212071679945, + "nauc_recall_at_5_diff1": 0.6061122817873875, + "nauc_recall_at_5_max": 0.4966926760848395, + "nauc_recall_at_5_std": 0.029252343168589257, + "ndcg_at_1": 0.51056, + "ndcg_at_10": 0.61418, + "ndcg_at_100": 0.6686, + "ndcg_at_1000": 0.67579, + "ndcg_at_20": 0.64538, + "ndcg_at_3": 0.55406, + "ndcg_at_5": 0.58532, + "precision_at_1": 0.51056, + "precision_at_10": 0.10211, + "precision_at_100": 0.01468, + "precision_at_1000": 0.00155, + "precision_at_20": 0.06144, + "precision_at_3": 0.24061, + "precision_at_5": 0.17254, + "recall_at_1": 0.44253, + "recall_at_10": 0.74843, + "recall_at_100": 0.95268, + "recall_at_1000": 1.0, + "recall_at_20": 0.85299, + "recall_at_3": 0.58457, + "recall_at_5": 0.66128 + } + ] + }, + "task_name": "LegalSummarization" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3-lite/no_revision_available/model_meta.json b/results/voyageai__voyage-3-lite/no_revision_available/model_meta.json new file mode 100644 index 000000000..d74dfe305 --- /dev/null +++ b/results/voyageai__voyage-3-lite/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "voyageai__voyage-3-lite", "revision": "no_revision_available", "release_date": "2024-09-22", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 32000, "embed_dim": 512, "license": null, "open_source": false, "similarity_fn_name": null, "framework": [], "loader": "VoyageWrapper"} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/AILACasedocs.json b/results/voyageai__voyage-3/no_revision_available/AILACasedocs.json new file mode 100644 index 000000000..960995985 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/AILACasedocs.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "4106e6bcc72e0698d714ea8b101355e3e238431a", + "evaluation_time": 15.335664987564087, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.4059, + "map_at_1": 0.13108, + "map_at_10": 0.31197, + "map_at_100": 0.35734, + "map_at_1000": 0.35952, + "map_at_20": 0.33371, + "map_at_3": 0.23493, + "map_at_5": 0.27606, + "mrr_at_1": 0.38, + "mrr_at_10": 0.47241269841269834, + "mrr_at_100": 0.4858829393650526, + "mrr_at_1000": 0.48608095916703276, + "mrr_at_20": 0.48259578287209864, + "mrr_at_3": 0.4433333333333333, + "mrr_at_5": 0.4603333333333333, + "nauc_map_at_1000_diff1": 0.16694895214236313, + "nauc_map_at_1000_max": -0.004851240813065252, + "nauc_map_at_1000_std": -0.3276023716108544, + "nauc_map_at_100_diff1": 0.16035759494427687, + "nauc_map_at_100_max": -0.0070968573952436675, + "nauc_map_at_100_std": -0.32759458907272154, + "nauc_map_at_10_diff1": 0.16429863419210805, + "nauc_map_at_10_max": -0.03555600851134987, + "nauc_map_at_10_std": -0.36193615969716103, + "nauc_map_at_1_diff1": 0.3156884343530487, + "nauc_map_at_1_max": -0.01914848988009138, + "nauc_map_at_1_std": -0.39439221352049947, + "nauc_map_at_20_diff1": 0.16514449718967777, + "nauc_map_at_20_max": -0.012509658780211808, + "nauc_map_at_20_std": -0.3386721619810786, + "nauc_map_at_3_diff1": 0.18409128922468598, + "nauc_map_at_3_max": -0.07883809639247819, + "nauc_map_at_3_std": -0.37851334879323995, + "nauc_map_at_5_diff1": 0.1936795902264733, + "nauc_map_at_5_max": -0.047901694845251884, + "nauc_map_at_5_std": -0.3519422667961817, + "nauc_mrr_at_1000_diff1": 0.1689760468430888, + "nauc_mrr_at_1000_max": -0.027320902940415996, + "nauc_mrr_at_1000_std": -0.3527906649053023, + "nauc_mrr_at_100_diff1": 0.16782829549610742, + "nauc_mrr_at_100_max": -0.026971289961236097, + "nauc_mrr_at_100_std": -0.35214431230626503, + "nauc_mrr_at_10_diff1": 0.16311548443737944, + "nauc_mrr_at_10_max": -0.03173249298615006, + "nauc_mrr_at_10_std": -0.3561114803025323, + "nauc_mrr_at_1_diff1": 0.26098822550879847, + "nauc_mrr_at_1_max": -0.008447482650170403, + "nauc_mrr_at_1_std": -0.415798092168534, + "nauc_mrr_at_20_diff1": 0.16695135284114054, + "nauc_mrr_at_20_max": -0.021842716362981473, + "nauc_mrr_at_20_std": -0.34530249739068836, + "nauc_mrr_at_3_diff1": 0.16257724183254463, + "nauc_mrr_at_3_max": -0.06290054383542913, + "nauc_mrr_at_3_std": -0.38979724156790446, + "nauc_mrr_at_5_diff1": 0.1684111173894683, + "nauc_mrr_at_5_max": -0.02846040573346144, + "nauc_mrr_at_5_std": -0.3509171213488723, + "nauc_ndcg_at_1000_diff1": 0.14859935912425132, + "nauc_ndcg_at_1000_max": 0.0051069620975705485, + "nauc_ndcg_at_1000_std": -0.29497500265414794, + "nauc_ndcg_at_100_diff1": 0.09761403203901907, + "nauc_ndcg_at_100_max": 0.007454330243260363, + "nauc_ndcg_at_100_std": -0.2760660358021823, + "nauc_ndcg_at_10_diff1": 0.14624730067184738, + "nauc_ndcg_at_10_max": -0.009832848753714112, + "nauc_ndcg_at_10_std": -0.32886039047241283, + "nauc_ndcg_at_1_diff1": 0.26098822550879847, + "nauc_ndcg_at_1_max": -0.008447482650170403, + "nauc_ndcg_at_1_std": -0.415798092168534, + "nauc_ndcg_at_20_diff1": 0.11552513068810936, + "nauc_ndcg_at_20_max": 0.03300915028385415, + "nauc_ndcg_at_20_std": -0.27883298506052523, + "nauc_ndcg_at_3_diff1": 0.2050114634897266, + "nauc_ndcg_at_3_max": -0.008901530616783752, + "nauc_ndcg_at_3_std": -0.3410685514850177, + "nauc_ndcg_at_5_diff1": 0.16989948909331898, + "nauc_ndcg_at_5_max": 0.006884947450195044, + "nauc_ndcg_at_5_std": -0.28316311706303915, + "nauc_precision_at_1000_diff1": 0.1554474518492121, + "nauc_precision_at_1000_max": 0.1970677523356786, + "nauc_precision_at_1000_std": 0.17954890049117328, + "nauc_precision_at_100_diff1": 0.0847321622207761, + "nauc_precision_at_100_max": 0.1881117617291982, + "nauc_precision_at_100_std": 0.1894636015325672, + "nauc_precision_at_10_diff1": 0.025603780189009693, + "nauc_precision_at_10_max": 0.06411737253529351, + "nauc_precision_at_10_std": -0.11356901178392241, + "nauc_precision_at_1_diff1": 0.26098822550879847, + "nauc_precision_at_1_max": -0.008447482650170403, + "nauc_precision_at_1_std": -0.415798092168534, + "nauc_precision_at_20_diff1": 0.032244691876378094, + "nauc_precision_at_20_max": 0.16759055034413106, + "nauc_precision_at_20_std": 0.04683904227897175, + "nauc_precision_at_3_diff1": 0.09991039898803565, + "nauc_precision_at_3_max": -0.042934696674221236, + "nauc_precision_at_3_std": -0.3017235018183733, + "nauc_precision_at_5_diff1": 0.07338076844887172, + "nauc_precision_at_5_max": 0.017393779223419375, + "nauc_precision_at_5_std": -0.16942467981297024, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": -0.9211282130825046, + "nauc_recall_at_100_max": 0.1534592572853052, + "nauc_recall_at_100_std": 0.2951465513586929, + "nauc_recall_at_10_diff1": 0.03134444472160857, + "nauc_recall_at_10_max": -0.02594181388014944, + "nauc_recall_at_10_std": -0.2862214867761663, + "nauc_recall_at_1_diff1": 0.3156884343530487, + "nauc_recall_at_1_max": -0.01914848988009138, + "nauc_recall_at_1_std": -0.39439221352049947, + "nauc_recall_at_20_diff1": -0.05024520637579575, + "nauc_recall_at_20_max": 0.18616544507713648, + "nauc_recall_at_20_std": -0.024519938932394268, + "nauc_recall_at_3_diff1": 0.10502172815189759, + "nauc_recall_at_3_max": -0.11682175151338935, + "nauc_recall_at_3_std": -0.3426629585451144, + "nauc_recall_at_5_diff1": 0.10858443781872582, + "nauc_recall_at_5_max": -0.030412156645454085, + "nauc_recall_at_5_std": -0.23469206840824788, + "ndcg_at_1": 0.38, + "ndcg_at_10": 0.4059, + "ndcg_at_100": 0.53713, + "ndcg_at_1000": 0.55098, + "ndcg_at_20": 0.45433, + "ndcg_at_3": 0.37372, + "ndcg_at_5": 0.38205, + "precision_at_1": 0.38, + "precision_at_10": 0.18, + "precision_at_100": 0.0362, + "precision_at_1000": 0.0039, + "precision_at_20": 0.115, + "precision_at_3": 0.31333, + "precision_at_5": 0.26, + "recall_at_1": 0.13108, + "recall_at_10": 0.45852, + "recall_at_100": 0.93523, + "recall_at_1000": 1.0, + "recall_at_20": 0.62621, + "recall_at_3": 0.27186, + "recall_at_5": 0.35951 + } + ] + }, + "task_name": "AILACasedocs" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/AILAStatutes.json b/results/voyageai__voyage-3/no_revision_available/AILAStatutes.json new file mode 100644 index 000000000..6518e4104 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/AILAStatutes.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e", + "evaluation_time": 2.6054527759552, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.42489, + "map_at_1": 0.089, + "map_at_10": 0.28164, + "map_at_100": 0.35698, + "map_at_1000": 0.35698, + "map_at_20": 0.32248, + "map_at_3": 0.19317, + "map_at_5": 0.25133, + "mrr_at_1": 0.38, + "mrr_at_10": 0.5846031746031746, + "mrr_at_100": 0.586031746031746, + "mrr_at_1000": 0.586031746031746, + "mrr_at_20": 0.586031746031746, + "mrr_at_3": 0.5433333333333333, + "mrr_at_5": 0.5683333333333332, + "nauc_map_at_1000_diff1": 0.2233696824831322, + "nauc_map_at_1000_max": 0.29256769400089566, + "nauc_map_at_1000_std": 0.3009813503319834, + "nauc_map_at_100_diff1": 0.2233696824831322, + "nauc_map_at_100_max": 0.29256769400089566, + "nauc_map_at_100_std": 0.3009813503319834, + "nauc_map_at_10_diff1": 0.21383856105557086, + "nauc_map_at_10_max": 0.2899339769507187, + "nauc_map_at_10_std": 0.3189509097372717, + "nauc_map_at_1_diff1": 0.1522086189269905, + "nauc_map_at_1_max": 0.26452561655494733, + "nauc_map_at_1_std": 0.2725734401676043, + "nauc_map_at_20_diff1": 0.23029163485463477, + "nauc_map_at_20_max": 0.30383687590788633, + "nauc_map_at_20_std": 0.3085466534543769, + "nauc_map_at_3_diff1": 0.2243882723028131, + "nauc_map_at_3_max": 0.30622729741943583, + "nauc_map_at_3_std": 0.2703265435938838, + "nauc_map_at_5_diff1": 0.22594340578302483, + "nauc_map_at_5_max": 0.31354735965216785, + "nauc_map_at_5_std": 0.29579771586234693, + "nauc_mrr_at_1000_diff1": 0.22324504324661806, + "nauc_mrr_at_1000_max": 0.35594824291868543, + "nauc_mrr_at_1000_std": 0.351623303406185, + "nauc_mrr_at_100_diff1": 0.22324504324661806, + "nauc_mrr_at_100_max": 0.35594824291868543, + "nauc_mrr_at_100_std": 0.351623303406185, + "nauc_mrr_at_10_diff1": 0.22590228733416595, + "nauc_mrr_at_10_max": 0.35789879177529876, + "nauc_mrr_at_10_std": 0.3543800286296604, + "nauc_mrr_at_1_diff1": 0.18558469576066328, + "nauc_mrr_at_1_max": 0.32329165908559265, + "nauc_mrr_at_1_std": 0.3017440802640813, + "nauc_mrr_at_20_diff1": 0.22324504324661806, + "nauc_mrr_at_20_max": 0.35594824291868543, + "nauc_mrr_at_20_std": 0.351623303406185, + "nauc_mrr_at_3_diff1": 0.2319222676707437, + "nauc_mrr_at_3_max": 0.3291132760306547, + "nauc_mrr_at_3_std": 0.314379764202378, + "nauc_mrr_at_5_diff1": 0.23513989370163002, + "nauc_mrr_at_5_max": 0.3742794144265209, + "nauc_mrr_at_5_std": 0.36036957606805703, + "nauc_ndcg_at_1000_diff1": 0.21838446658827596, + "nauc_ndcg_at_1000_max": 0.29456600531595256, + "nauc_ndcg_at_1000_std": 0.3102662789532349, + "nauc_ndcg_at_100_diff1": 0.21838446658827596, + "nauc_ndcg_at_100_max": 0.29456600531595256, + "nauc_ndcg_at_100_std": 0.3102662789532349, + "nauc_ndcg_at_10_diff1": 0.24001982282652284, + "nauc_ndcg_at_10_max": 0.30712999586947326, + "nauc_ndcg_at_10_std": 0.3443949355923366, + "nauc_ndcg_at_1_diff1": 0.18558469576066328, + "nauc_ndcg_at_1_max": 0.32329165908559265, + "nauc_ndcg_at_1_std": 0.3017440802640813, + "nauc_ndcg_at_20_diff1": 0.2505051373172353, + "nauc_ndcg_at_20_max": 0.31519284968724237, + "nauc_ndcg_at_20_std": 0.31876820185786536, + "nauc_ndcg_at_3_diff1": 0.22000371911597214, + "nauc_ndcg_at_3_max": 0.2973729364565732, + "nauc_ndcg_at_3_std": 0.25293154567966375, + "nauc_ndcg_at_5_diff1": 0.25822713209824016, + "nauc_ndcg_at_5_max": 0.3558669525441037, + "nauc_ndcg_at_5_std": 0.3140017253084835, + "nauc_precision_at_1000_diff1": -0.27650427178426984, + "nauc_precision_at_1000_max": -0.3776732954025717, + "nauc_precision_at_1000_std": -0.14747260386747255, + "nauc_precision_at_100_diff1": -0.27650427178426606, + "nauc_precision_at_100_max": -0.3776732954025681, + "nauc_precision_at_100_std": -0.14747260386747074, + "nauc_precision_at_10_diff1": 0.1333324795410009, + "nauc_precision_at_10_max": 0.10122561889271679, + "nauc_precision_at_10_std": 0.2590064418631457, + "nauc_precision_at_1_diff1": 0.18558469576066328, + "nauc_precision_at_1_max": 0.32329165908559265, + "nauc_precision_at_1_std": 0.3017440802640813, + "nauc_precision_at_20_diff1": 0.1471377940070884, + "nauc_precision_at_20_max": 0.110342605520353, + "nauc_precision_at_20_std": 0.13634410911824726, + "nauc_precision_at_3_diff1": 0.20136817044872382, + "nauc_precision_at_3_max": 0.25430190157610805, + "nauc_precision_at_3_std": 0.2154868456739897, + "nauc_precision_at_5_diff1": 0.19706506084215056, + "nauc_precision_at_5_max": 0.2645187629707751, + "nauc_precision_at_5_std": 0.25802061715374386, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.2519359850648195, + "nauc_recall_at_10_max": 0.23569888907312178, + "nauc_recall_at_10_std": 0.3439052549482002, + "nauc_recall_at_1_diff1": 0.1522086189269905, + "nauc_recall_at_1_max": 0.26452561655494733, + "nauc_recall_at_1_std": 0.2725734401676043, + "nauc_recall_at_20_diff1": 0.2597034687841382, + "nauc_recall_at_20_max": 0.23025904542032347, + "nauc_recall_at_20_std": 0.23570816599197614, + "nauc_recall_at_3_diff1": 0.2902108760481054, + "nauc_recall_at_3_max": 0.32563269195590994, + "nauc_recall_at_3_std": 0.2147600471354974, + "nauc_recall_at_5_diff1": 0.29041095423347935, + "nauc_recall_at_5_max": 0.3681668161190342, + "nauc_recall_at_5_std": 0.2745769588772995, + "ndcg_at_1": 0.38, + "ndcg_at_10": 0.42489, + "ndcg_at_100": 0.60587, + "ndcg_at_1000": 0.60587, + "ndcg_at_20": 0.50515, + "ndcg_at_3": 0.37451, + "ndcg_at_5": 0.36821, + "precision_at_1": 0.38, + "precision_at_10": 0.204, + "precision_at_100": 0.0434, + "precision_at_1000": 0.00434, + "precision_at_20": 0.146, + "precision_at_3": 0.36, + "precision_at_5": 0.312, + "recall_at_1": 0.089, + "recall_at_10": 0.472, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.668, + "recall_at_3": 0.25467, + "recall_at_5": 0.36533 + } + ] + }, + "task_name": "AILAStatutes" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/AlloprofRetrieval.json b/results/voyageai__voyage-3/no_revision_available/AlloprofRetrieval.json new file mode 100644 index 000000000..4203fd087 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/AlloprofRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "fcf295ea64c750f41fadbaa37b9b861558e1bfbd", + "evaluation_time": 44.84511041641235, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "fra-Latn" + ], + "main_score": 0.73762, + "map_at_1": 0.5462, + "map_at_10": 0.67833, + "map_at_100": 0.68132, + "map_at_1000": 0.6814, + "map_at_20": 0.68038, + "map_at_3": 0.65206, + "map_at_5": 0.66983, + "mrr_at_1": 0.5462003454231433, + "mrr_at_10": 0.6783330934561503, + "mrr_at_100": 0.6813216843444168, + "mrr_at_1000": 0.6813984427556294, + "mrr_at_20": 0.6803799812092168, + "mrr_at_3": 0.6520581462291298, + "mrr_at_5": 0.6698258491652274, + "nauc_map_at_1000_diff1": 0.43723801793631956, + "nauc_map_at_1000_max": 0.2897099342855824, + "nauc_map_at_1000_std": -0.0160998338268507, + "nauc_map_at_100_diff1": 0.437225340598113, + "nauc_map_at_100_max": 0.2898486343897764, + "nauc_map_at_100_std": -0.015961636200445346, + "nauc_map_at_10_diff1": 0.43674333817126, + "nauc_map_at_10_max": 0.2915118501310887, + "nauc_map_at_10_std": -0.015434704656535219, + "nauc_map_at_1_diff1": 0.4592036600458886, + "nauc_map_at_1_max": 0.25616967041085037, + "nauc_map_at_1_std": -0.032602831195759845, + "nauc_map_at_20_diff1": 0.4371595381591378, + "nauc_map_at_20_max": 0.2908460860537897, + "nauc_map_at_20_std": -0.015434403252826344, + "nauc_map_at_3_diff1": 0.4289355009813948, + "nauc_map_at_3_max": 0.2790366987413791, + "nauc_map_at_3_std": -0.025192825672504277, + "nauc_map_at_5_diff1": 0.43525363838679154, + "nauc_map_at_5_max": 0.2908755999859609, + "nauc_map_at_5_std": -0.018710130092887953, + "nauc_mrr_at_1000_diff1": 0.43723801793631956, + "nauc_mrr_at_1000_max": 0.2897099342855824, + "nauc_mrr_at_1000_std": -0.0160998338268507, + "nauc_mrr_at_100_diff1": 0.437225340598113, + "nauc_mrr_at_100_max": 0.2898486343897764, + "nauc_mrr_at_100_std": -0.015961636200445346, + "nauc_mrr_at_10_diff1": 0.43674333817126, + "nauc_mrr_at_10_max": 0.2915118501310887, + "nauc_mrr_at_10_std": -0.015434704656535219, + "nauc_mrr_at_1_diff1": 0.4592036600458886, + "nauc_mrr_at_1_max": 0.25616967041085037, + "nauc_mrr_at_1_std": -0.032602831195759845, + "nauc_mrr_at_20_diff1": 0.4371595381591378, + "nauc_mrr_at_20_max": 0.2908460860537897, + "nauc_mrr_at_20_std": -0.015434403252826344, + "nauc_mrr_at_3_diff1": 0.4289355009813948, + "nauc_mrr_at_3_max": 0.2790366987413791, + "nauc_mrr_at_3_std": -0.025192825672504277, + "nauc_mrr_at_5_diff1": 0.43525363838679154, + "nauc_mrr_at_5_max": 0.2908755999859609, + "nauc_mrr_at_5_std": -0.018710130092887953, + "nauc_ndcg_at_1000_diff1": 0.4356346978498041, + "nauc_ndcg_at_1000_max": 0.3004061448812192, + "nauc_ndcg_at_1000_std": -0.008114310677887357, + "nauc_ndcg_at_100_diff1": 0.43489193868391396, + "nauc_ndcg_at_100_max": 0.3045949253994621, + "nauc_ndcg_at_100_std": -0.004203242087672975, + "nauc_ndcg_at_10_diff1": 0.43262209112936423, + "nauc_ndcg_at_10_max": 0.31622630962387477, + "nauc_ndcg_at_10_std": 0.0014503268672549435, + "nauc_ndcg_at_1_diff1": 0.4592036600458886, + "nauc_ndcg_at_1_max": 0.25616967041085037, + "nauc_ndcg_at_1_std": -0.032602831195759845, + "nauc_ndcg_at_20_diff1": 0.4346677074806468, + "nauc_ndcg_at_20_max": 0.314192252981524, + "nauc_ndcg_at_20_std": 0.001866257948997183, + "nauc_ndcg_at_3_diff1": 0.418215118115295, + "nauc_ndcg_at_3_max": 0.2873906311219984, + "nauc_ndcg_at_3_std": -0.023492382044289682, + "nauc_ndcg_at_5_diff1": 0.4292477948512114, + "nauc_ndcg_at_5_max": 0.31254215888202613, + "nauc_ndcg_at_5_std": -0.009291629246894488, + "nauc_precision_at_1000_diff1": 0.9564186519138237, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.3981830669583472, + "nauc_precision_at_100_max": 0.7779361614640738, + "nauc_precision_at_100_std": 0.4629734064729158, + "nauc_precision_at_10_diff1": 0.4031174401431326, + "nauc_precision_at_10_max": 0.5594404044497714, + "nauc_precision_at_10_std": 0.17825791430867527, + "nauc_precision_at_1_diff1": 0.4592036600458886, + "nauc_precision_at_1_max": 0.25616967041085037, + "nauc_precision_at_1_std": -0.032602831195759845, + "nauc_precision_at_20_diff1": 0.42154938969870864, + "nauc_precision_at_20_max": 0.6703388349409724, + "nauc_precision_at_20_std": 0.28746106940135646, + "nauc_precision_at_3_diff1": 0.37636084645542517, + "nauc_precision_at_3_max": 0.3201253277106461, + "nauc_precision_at_3_std": -0.017286652690601464, + "nauc_precision_at_5_diff1": 0.3994162866319014, + "nauc_precision_at_5_max": 0.43806311888658356, + "nauc_precision_at_5_std": 0.0465219093082239, + "nauc_recall_at_1000_diff1": 0.9564186519138445, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": 0.39818306695834405, + "nauc_recall_at_100_max": 0.7779361614640701, + "nauc_recall_at_100_std": 0.46297340647290347, + "nauc_recall_at_10_diff1": 0.40311744014313, + "nauc_recall_at_10_max": 0.5594404044497706, + "nauc_recall_at_10_std": 0.17825791430867255, + "nauc_recall_at_1_diff1": 0.4592036600458886, + "nauc_recall_at_1_max": 0.25616967041085037, + "nauc_recall_at_1_std": -0.032602831195759845, + "nauc_recall_at_20_diff1": 0.4215493896987037, + "nauc_recall_at_20_max": 0.6703388349409644, + "nauc_recall_at_20_std": 0.28746106940135, + "nauc_recall_at_3_diff1": 0.3763608464554263, + "nauc_recall_at_3_max": 0.32012532771064733, + "nauc_recall_at_3_std": -0.01728665269059916, + "nauc_recall_at_5_diff1": 0.39941628663189893, + "nauc_recall_at_5_max": 0.43806311888658345, + "nauc_recall_at_5_std": 0.04652190930822123, + "ndcg_at_1": 0.5462, + "ndcg_at_10": 0.73762, + "ndcg_at_100": 0.75157, + "ndcg_at_1000": 0.75339, + "ndcg_at_20": 0.74506, + "ndcg_at_3": 0.6854, + "ndcg_at_5": 0.71714, + "precision_at_1": 0.5462, + "precision_at_10": 0.0921, + "precision_at_100": 0.00985, + "precision_at_1000": 0.001, + "precision_at_20": 0.04752, + "precision_at_3": 0.26051, + "precision_at_5": 0.17159, + "recall_at_1": 0.5462, + "recall_at_10": 0.92098, + "recall_at_100": 0.98489, + "recall_at_1000": 0.9987, + "recall_at_20": 0.95035, + "recall_at_3": 0.78152, + "recall_at_5": 0.85794 + } + ] + }, + "task_name": "AlloprofRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/AppsRetrieval.json b/results/voyageai__voyage-3/no_revision_available/AppsRetrieval.json new file mode 100644 index 000000000..99dbf9a30 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/AppsRetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "f22508f96b7a36c2415181ed8bb76f76e04ae2d5", + "evaluation_time": 85.59606695175171, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ], + "main_score": 0.7303, + "map_at_1": 0.6008, + "map_at_10": 0.68884, + "map_at_100": 0.69385, + "map_at_1000": 0.69399, + "map_at_20": 0.6921, + "map_at_3": 0.67021, + "map_at_5": 0.68099, + "mrr_at_1": 0.601062416998672, + "mrr_at_10": 0.6889654082084365, + "mrr_at_100": 0.6939780026959514, + "mrr_at_1000": 0.6941190387796582, + "mrr_at_20": 0.6922277423531847, + "mrr_at_3": 0.6703408587870746, + "mrr_at_5": 0.6811243913235949, + "nauc_map_at_1000_diff1": 0.740248229999432, + "nauc_map_at_1000_max": 0.4605852355665703, + "nauc_map_at_1000_std": -0.13638494500719864, + "nauc_map_at_100_diff1": 0.7401701465956654, + "nauc_map_at_100_max": 0.4606070340781131, + "nauc_map_at_100_std": -0.13621964766262948, + "nauc_map_at_10_diff1": 0.7387051155016687, + "nauc_map_at_10_max": 0.4610572119099232, + "nauc_map_at_10_std": -0.13583029294686066, + "nauc_map_at_1_diff1": 0.7755589109478567, + "nauc_map_at_1_max": 0.4474158651825035, + "nauc_map_at_1_std": -0.15430108939050924, + "nauc_map_at_20_diff1": 0.7393398397088838, + "nauc_map_at_20_max": 0.4604182241337643, + "nauc_map_at_20_std": -0.13627263043646248, + "nauc_map_at_3_diff1": 0.7411933263933311, + "nauc_map_at_3_max": 0.45999791403590307, + "nauc_map_at_3_std": -0.14424082717771625, + "nauc_map_at_5_diff1": 0.7383509573003422, + "nauc_map_at_5_max": 0.46190253066558395, + "nauc_map_at_5_std": -0.13844993152057194, + "nauc_mrr_at_1000_diff1": 0.7399630232144186, + "nauc_mrr_at_1000_max": 0.46053522018659865, + "nauc_mrr_at_1000_std": -0.1366062142890654, + "nauc_mrr_at_100_diff1": 0.7398850778134987, + "nauc_mrr_at_100_max": 0.46055705245347306, + "nauc_mrr_at_100_std": -0.13644074907277168, + "nauc_mrr_at_10_diff1": 0.738424784995603, + "nauc_mrr_at_10_max": 0.46100824797124834, + "nauc_mrr_at_10_std": -0.13604731599408065, + "nauc_mrr_at_1_diff1": 0.775076545046777, + "nauc_mrr_at_1_max": 0.44732270301802546, + "nauc_mrr_at_1_std": -0.15467631071724744, + "nauc_mrr_at_20_diff1": 0.7390565079115019, + "nauc_mrr_at_20_max": 0.460368535499979, + "nauc_mrr_at_20_std": -0.13649222662109814, + "nauc_mrr_at_3_diff1": 0.740905885514203, + "nauc_mrr_at_3_max": 0.45994759394253387, + "nauc_mrr_at_3_std": -0.1444544756553248, + "nauc_mrr_at_5_diff1": 0.7380798604090736, + "nauc_mrr_at_5_max": 0.46184385845608855, + "nauc_mrr_at_5_std": -0.13867816722454912, + "nauc_ndcg_at_1000_diff1": 0.7337145803699073, + "nauc_ndcg_at_1000_max": 0.46305068814310507, + "nauc_ndcg_at_1000_std": -0.12907327505835237, + "nauc_ndcg_at_100_diff1": 0.7317774195941744, + "nauc_ndcg_at_100_max": 0.4636195474822238, + "nauc_ndcg_at_100_std": -0.12437785867598594, + "nauc_ndcg_at_10_diff1": 0.7233055430148577, + "nauc_ndcg_at_10_max": 0.4650271635806324, + "nauc_ndcg_at_10_std": -0.12334883020386732, + "nauc_ndcg_at_1_diff1": 0.7755589109478567, + "nauc_ndcg_at_1_max": 0.4474158651825035, + "nauc_ndcg_at_1_std": -0.15430108939050924, + "nauc_ndcg_at_20_diff1": 0.7254156497516492, + "nauc_ndcg_at_20_max": 0.462509045568244, + "nauc_ndcg_at_20_std": -0.12464777730390277, + "nauc_ndcg_at_3_diff1": 0.7296518125878367, + "nauc_ndcg_at_3_max": 0.46314375656976087, + "nauc_ndcg_at_3_std": -0.14220199070080916, + "nauc_ndcg_at_5_diff1": 0.7233673838844502, + "nauc_ndcg_at_5_max": 0.46671409206898246, + "nauc_ndcg_at_5_std": -0.13115163267403424, + "nauc_precision_at_1000_diff1": 0.6539905311169442, + "nauc_precision_at_1000_max": 0.7574966236318875, + "nauc_precision_at_1000_std": 0.5548581004954547, + "nauc_precision_at_100_diff1": 0.6240699875925971, + "nauc_precision_at_100_max": 0.523427974477694, + "nauc_precision_at_100_std": 0.19441025608685397, + "nauc_precision_at_10_diff1": 0.6375592097902707, + "nauc_precision_at_10_max": 0.4856910298374126, + "nauc_precision_at_10_std": -0.04716524701564442, + "nauc_precision_at_1_diff1": 0.7755589109478567, + "nauc_precision_at_1_max": 0.4474158651825035, + "nauc_precision_at_1_std": -0.15430108939050924, + "nauc_precision_at_20_diff1": 0.6195408294710768, + "nauc_precision_at_20_max": 0.47154683261534946, + "nauc_precision_at_20_std": -0.025475318430463918, + "nauc_precision_at_3_diff1": 0.6889991809114094, + "nauc_precision_at_3_max": 0.473766510740177, + "nauc_precision_at_3_std": -0.1355887146484221, + "nauc_precision_at_5_diff1": 0.660908851625188, + "nauc_precision_at_5_max": 0.4862544393901494, + "nauc_precision_at_5_std": -0.0999221838903806, + "nauc_recall_at_1000_diff1": 0.6539905311169901, + "nauc_recall_at_1000_max": 0.7574966236319249, + "nauc_recall_at_1000_std": 0.5548581004955162, + "nauc_recall_at_100_diff1": 0.6240699875925984, + "nauc_recall_at_100_max": 0.5234279744777011, + "nauc_recall_at_100_std": 0.1944102560868612, + "nauc_recall_at_10_diff1": 0.6375592097902713, + "nauc_recall_at_10_max": 0.48569102983741247, + "nauc_recall_at_10_std": -0.047165247015642134, + "nauc_recall_at_1_diff1": 0.7755589109478567, + "nauc_recall_at_1_max": 0.4474158651825035, + "nauc_recall_at_1_std": -0.15430108939050924, + "nauc_recall_at_20_diff1": 0.6195408294710782, + "nauc_recall_at_20_max": 0.47154683261534946, + "nauc_recall_at_20_std": -0.02547531843046639, + "nauc_recall_at_3_diff1": 0.6889991809114084, + "nauc_recall_at_3_max": 0.473766510740176, + "nauc_recall_at_3_std": -0.13558871464842304, + "nauc_recall_at_5_diff1": 0.6609088516251881, + "nauc_recall_at_5_max": 0.4862544393901485, + "nauc_recall_at_5_std": -0.0999221838903799, + "ndcg_at_1": 0.6008, + "ndcg_at_10": 0.7303, + "ndcg_at_100": 0.75427, + "ndcg_at_1000": 0.75767, + "ndcg_at_20": 0.74215, + "ndcg_at_3": 0.69195, + "ndcg_at_5": 0.71151, + "precision_at_1": 0.6008, + "precision_at_10": 0.086, + "precision_at_100": 0.00971, + "precision_at_1000": 0.001, + "precision_at_20": 0.04534, + "precision_at_3": 0.25153, + "precision_at_5": 0.16048, + "recall_at_1": 0.6008, + "recall_at_10": 0.86003, + "recall_at_100": 0.97105, + "recall_at_1000": 0.99708, + "recall_at_20": 0.90677, + "recall_at_3": 0.75458, + "recall_at_5": 0.80239 + } + ] + }, + "task_name": "AppsRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/BSARDRetrieval.json b/results/voyageai__voyage-3/no_revision_available/BSARDRetrieval.json new file mode 100644 index 000000000..a84c5dd69 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/BSARDRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "5effa1b9b5fa3b0f9e12523e6e43e5f86a6e6d59", + "evaluation_time": 91.91388154029846, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "fra-Latn" + ], + "main_score": 0.82883, + "map_at_1": 0.2973, + "map_at_10": 0.39574, + "map_at_100": 0.40473, + "map_at_1000": 0.40528, + "map_at_20": 0.40114, + "map_at_3": 0.36862, + "map_at_5": 0.38371, + "mrr_at_1": 0.2972972972972973, + "mrr_at_10": 0.39573502073502087, + "mrr_at_100": 0.4047277913221853, + "mrr_at_1000": 0.4052776284728539, + "mrr_at_20": 0.4011394300867987, + "mrr_at_3": 0.36861861861861867, + "mrr_at_5": 0.38370870870870877, + "nauc_map_at_1000_diff1": 0.31152287617037255, + "nauc_map_at_1000_max": 0.22132396049082534, + "nauc_map_at_1000_std": -0.06236878697982136, + "nauc_map_at_100_diff1": 0.31129349446927324, + "nauc_map_at_100_max": 0.2210387216611584, + "nauc_map_at_100_std": -0.06221995568226558, + "nauc_map_at_10_diff1": 0.3039600488573116, + "nauc_map_at_10_max": 0.2225862850202198, + "nauc_map_at_10_std": -0.062014219558287745, + "nauc_map_at_1_diff1": 0.3472737595766608, + "nauc_map_at_1_max": 0.1864640093437044, + "nauc_map_at_1_std": -0.07561112074547974, + "nauc_map_at_20_diff1": 0.3104020893459893, + "nauc_map_at_20_max": 0.22151328532748105, + "nauc_map_at_20_std": -0.0636038409978758, + "nauc_map_at_3_diff1": 0.3318347534286368, + "nauc_map_at_3_max": 0.22940632410692968, + "nauc_map_at_3_std": -0.07292308648398768, + "nauc_map_at_5_diff1": 0.31816882845753736, + "nauc_map_at_5_max": 0.22911385940458712, + "nauc_map_at_5_std": -0.07327193036552977, + "nauc_mrr_at_1000_diff1": 0.31152287617037255, + "nauc_mrr_at_1000_max": 0.22132396049082534, + "nauc_mrr_at_1000_std": -0.06236878697982136, + "nauc_mrr_at_100_diff1": 0.31129349446927324, + "nauc_mrr_at_100_max": 0.2210387216611584, + "nauc_mrr_at_100_std": -0.06221995568226558, + "nauc_mrr_at_10_diff1": 0.3039600488573116, + "nauc_mrr_at_10_max": 0.2225862850202198, + "nauc_mrr_at_10_std": -0.062014219558287745, + "nauc_mrr_at_1_diff1": 0.3472737595766608, + "nauc_mrr_at_1_max": 0.1864640093437044, + "nauc_mrr_at_1_std": -0.07561112074547974, + "nauc_mrr_at_20_diff1": 0.3104020893459893, + "nauc_mrr_at_20_max": 0.22151328532748105, + "nauc_mrr_at_20_std": -0.0636038409978758, + "nauc_mrr_at_3_diff1": 0.3318347534286368, + "nauc_mrr_at_3_max": 0.22940632410692968, + "nauc_mrr_at_3_std": -0.07292308648398768, + "nauc_mrr_at_5_diff1": 0.31816882845753736, + "nauc_mrr_at_5_max": 0.22911385940458712, + "nauc_mrr_at_5_std": -0.07327193036552977, + "nauc_ndcg_at_1000_diff1": 0.3040629439883405, + "nauc_ndcg_at_1000_max": 0.22209619311469206, + "nauc_ndcg_at_1000_std": -0.04720811028143234, + "nauc_ndcg_at_100_diff1": 0.29630621434686727, + "nauc_ndcg_at_100_max": 0.21432294409348732, + "nauc_ndcg_at_100_std": -0.03726209419069324, + "nauc_ndcg_at_10_diff1": 0.2652998041650886, + "nauc_ndcg_at_10_max": 0.22259060517114435, + "nauc_ndcg_at_10_std": -0.04443013297252156, + "nauc_ndcg_at_1_diff1": 0.3472737595766608, + "nauc_ndcg_at_1_max": 0.1864640093437044, + "nauc_ndcg_at_1_std": -0.07561112074547974, + "nauc_ndcg_at_20_diff1": 0.2878380362490949, + "nauc_ndcg_at_20_max": 0.22017318997571164, + "nauc_ndcg_at_20_std": -0.049228964876875676, + "nauc_ndcg_at_3_diff1": 0.3254029480309303, + "nauc_ndcg_at_3_max": 0.23702925635109418, + "nauc_ndcg_at_3_std": -0.07289431309270122, + "nauc_ndcg_at_5_diff1": 0.30043258795807976, + "nauc_ndcg_at_5_max": 0.23613392453708743, + "nauc_ndcg_at_5_std": -0.07360409529700572, + "nauc_precision_at_1000_diff1": 0.5049056346405278, + "nauc_precision_at_1000_max": 0.30667933919671164, + "nauc_precision_at_1000_std": 0.13510984018473832, + "nauc_precision_at_100_diff1": 0.24637437648654864, + "nauc_precision_at_100_max": 0.1331578071487979, + "nauc_precision_at_100_std": 0.17685366278688847, + "nauc_precision_at_10_diff1": 0.12002091014393737, + "nauc_precision_at_10_max": 0.21655576619219505, + "nauc_precision_at_10_std": 0.026736222739340936, + "nauc_precision_at_1_diff1": 0.3472737595766608, + "nauc_precision_at_1_max": 0.1864640093437044, + "nauc_precision_at_1_std": -0.07561112074547974, + "nauc_precision_at_20_diff1": 0.20469850259884514, + "nauc_precision_at_20_max": 0.2079271530151885, + "nauc_precision_at_20_std": 0.01563516050807325, + "nauc_precision_at_3_diff1": 0.3065812608968494, + "nauc_precision_at_3_max": 0.25696320759106567, + "nauc_precision_at_3_std": -0.07306233535695753, + "nauc_precision_at_5_diff1": 0.2440070220853028, + "nauc_precision_at_5_max": 0.254425447543157, + "nauc_precision_at_5_std": -0.07500125151569056, + "nauc_recall_at_1000_diff1": 0.5049056346405323, + "nauc_recall_at_1000_max": 0.3066793391967186, + "nauc_recall_at_1000_std": 0.13510984018474154, + "nauc_recall_at_100_diff1": 0.24637437648654725, + "nauc_recall_at_100_max": 0.1331578071487963, + "nauc_recall_at_100_std": 0.1768536627868875, + "nauc_recall_at_10_diff1": 0.12002091014393755, + "nauc_recall_at_10_max": 0.21655576619219458, + "nauc_recall_at_10_std": 0.02673622273934152, + "nauc_recall_at_1_diff1": 0.3472737595766608, + "nauc_recall_at_1_max": 0.1864640093437044, + "nauc_recall_at_1_std": -0.07561112074547974, + "nauc_recall_at_20_diff1": 0.2046985025988449, + "nauc_recall_at_20_max": 0.20792715301518847, + "nauc_recall_at_20_std": 0.01563516050807367, + "nauc_recall_at_3_diff1": 0.30658126089685, + "nauc_recall_at_3_max": 0.25696320759106633, + "nauc_recall_at_3_std": -0.07306233535695683, + "nauc_recall_at_5_diff1": 0.24400702208530287, + "nauc_recall_at_5_max": 0.25442544754315727, + "nauc_recall_at_5_std": -0.07500125151569063, + "ndcg_at_1": 0.2973, + "ndcg_at_10": 0.44872, + "ndcg_at_100": 0.49343, + "ndcg_at_1000": 0.50867, + "ndcg_at_20": 0.46745, + "ndcg_at_3": 0.39302, + "ndcg_at_5": 0.41958, + "precision_at_1": 0.2973, + "precision_at_10": 0.06171, + "precision_at_100": 0.00829, + "precision_at_1000": 0.00095, + "precision_at_20": 0.03446, + "precision_at_3": 0.15465, + "precision_at_5": 0.10541, + "recall_at_1": 0.2973, + "recall_at_10": 0.61712, + "recall_at_100": 0.82883, + "recall_at_1000": 0.95045, + "recall_at_20": 0.68919, + "recall_at_3": 0.46396, + "recall_at_5": 0.52703 + } + ] + }, + "task_name": "BSARDRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CodeFeedbackMT.json b/results/voyageai__voyage-3/no_revision_available/CodeFeedbackMT.json new file mode 100644 index 000000000..de8a25e9a --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CodeFeedbackMT.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f", + "evaluation_time": 640.0915267467499, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6669, + "map_at_1": 0.54297, + "map_at_10": 0.62691, + "map_at_100": 0.6324, + "map_at_1000": 0.63269, + "map_at_20": 0.63028, + "map_at_3": 0.60856, + "map_at_5": 0.61946, + "mrr_at_1": 0.5429690442117948, + "mrr_at_10": 0.6269120665765193, + "mrr_at_100": 0.6324041762120336, + "mrr_at_1000": 0.632686585899261, + "mrr_at_20": 0.6302804848597193, + "mrr_at_3": 0.6085586603399308, + "mrr_at_5": 0.61946473852024, + "nauc_map_at_1000_diff1": 0.6833003894555225, + "nauc_map_at_1000_max": 0.33316843814852964, + "nauc_map_at_1000_std": -0.27632127290149994, + "nauc_map_at_100_diff1": 0.6832086399350786, + "nauc_map_at_100_max": 0.3332163200886008, + "nauc_map_at_100_std": -0.2763250221173925, + "nauc_map_at_10_diff1": 0.6823678435064746, + "nauc_map_at_10_max": 0.3331495104790356, + "nauc_map_at_10_std": -0.27802880100242927, + "nauc_map_at_1_diff1": 0.7119883438978566, + "nauc_map_at_1_max": 0.31495002698850766, + "nauc_map_at_1_std": -0.2645224682310068, + "nauc_map_at_20_diff1": 0.6829470330634594, + "nauc_map_at_20_max": 0.333302199572596, + "nauc_map_at_20_std": -0.27707914016713864, + "nauc_map_at_3_diff1": 0.6857295861961805, + "nauc_map_at_3_max": 0.3329735402039582, + "nauc_map_at_3_std": -0.2798270689379594, + "nauc_map_at_5_diff1": 0.683657907821838, + "nauc_map_at_5_max": 0.3330379135438407, + "nauc_map_at_5_std": -0.2803768833260438, + "nauc_mrr_at_1000_diff1": 0.6833003186978719, + "nauc_mrr_at_1000_max": 0.33316844628824277, + "nauc_mrr_at_1000_std": -0.2763212576091518, + "nauc_mrr_at_100_diff1": 0.6832085691101638, + "nauc_mrr_at_100_max": 0.33321632861622696, + "nauc_mrr_at_100_std": -0.2763250067125721, + "nauc_mrr_at_10_diff1": 0.6823678435064746, + "nauc_mrr_at_10_max": 0.3331495104790356, + "nauc_mrr_at_10_std": -0.27802880100242927, + "nauc_mrr_at_1_diff1": 0.7119883438978566, + "nauc_mrr_at_1_max": 0.31495002698850766, + "nauc_mrr_at_1_std": -0.2645224682310068, + "nauc_mrr_at_20_diff1": 0.6829470330634594, + "nauc_mrr_at_20_max": 0.333302199572596, + "nauc_mrr_at_20_std": -0.27707914016713864, + "nauc_mrr_at_3_diff1": 0.6857295861961805, + "nauc_mrr_at_3_max": 0.3329735402039582, + "nauc_mrr_at_3_std": -0.2798270689379594, + "nauc_mrr_at_5_diff1": 0.683657907821838, + "nauc_mrr_at_5_max": 0.3330379135438407, + "nauc_mrr_at_5_std": -0.2803768833260438, + "nauc_ndcg_at_1000_diff1": 0.6745701809860494, + "nauc_ndcg_at_1000_max": 0.3384858623917956, + "nauc_ndcg_at_1000_std": -0.27076004853810776, + "nauc_ndcg_at_100_diff1": 0.6718521770176635, + "nauc_ndcg_at_100_max": 0.3403011041725673, + "nauc_ndcg_at_100_std": -0.26933971850932165, + "nauc_ndcg_at_10_diff1": 0.6681676894990839, + "nauc_ndcg_at_10_max": 0.3395165335125083, + "nauc_ndcg_at_10_std": -0.28006918050964935, + "nauc_ndcg_at_1_diff1": 0.7119883438978566, + "nauc_ndcg_at_1_max": 0.31495002698850766, + "nauc_ndcg_at_1_std": -0.2645224682310068, + "nauc_ndcg_at_20_diff1": 0.6699235308368756, + "nauc_ndcg_at_20_max": 0.34057453720788744, + "nauc_ndcg_at_20_std": -0.2759797665504346, + "nauc_ndcg_at_3_diff1": 0.676393811987517, + "nauc_ndcg_at_3_max": 0.33856968787053254, + "nauc_ndcg_at_3_std": -0.28485983656549974, + "nauc_ndcg_at_5_diff1": 0.6720282076112493, + "nauc_ndcg_at_5_max": 0.3387062284885767, + "nauc_ndcg_at_5_std": -0.28623287980398937, + "nauc_precision_at_1000_diff1": 0.4617333567670125, + "nauc_precision_at_1000_max": 0.5038683041808043, + "nauc_precision_at_1000_std": 0.21310469411427507, + "nauc_precision_at_100_diff1": 0.5642707784451954, + "nauc_precision_at_100_max": 0.41872829956364, + "nauc_precision_at_100_std": -0.12813146903091197, + "nauc_precision_at_10_diff1": 0.6031704022409506, + "nauc_precision_at_10_max": 0.3674417347178129, + "nauc_precision_at_10_std": -0.28514923221295924, + "nauc_precision_at_1_diff1": 0.7119883438978566, + "nauc_precision_at_1_max": 0.31495002698850766, + "nauc_precision_at_1_std": -0.2645224682310068, + "nauc_precision_at_20_diff1": 0.5975489978739603, + "nauc_precision_at_20_max": 0.3825461744551741, + "nauc_precision_at_20_std": -0.2581091725677252, + "nauc_precision_at_3_diff1": 0.6455323594330408, + "nauc_precision_at_3_max": 0.3567816344156971, + "nauc_precision_at_3_std": -0.30135216051748437, + "nauc_precision_at_5_diff1": 0.6284727888552459, + "nauc_precision_at_5_max": 0.3590357637256036, + "nauc_precision_at_5_std": -0.3079306794050018, + "nauc_recall_at_1000_diff1": 0.46173335676701005, + "nauc_recall_at_1000_max": 0.5038683041807919, + "nauc_recall_at_1000_std": 0.21310469411427502, + "nauc_recall_at_100_diff1": 0.5642707784451941, + "nauc_recall_at_100_max": 0.4187282995636387, + "nauc_recall_at_100_std": -0.1281314690309122, + "nauc_recall_at_10_diff1": 0.6031704022409503, + "nauc_recall_at_10_max": 0.3674417347178134, + "nauc_recall_at_10_std": -0.2851492322129584, + "nauc_recall_at_1_diff1": 0.7119883438978566, + "nauc_recall_at_1_max": 0.31495002698850766, + "nauc_recall_at_1_std": -0.2645224682310068, + "nauc_recall_at_20_diff1": 0.5975489978739608, + "nauc_recall_at_20_max": 0.3825461744551748, + "nauc_recall_at_20_std": -0.258109172567725, + "nauc_recall_at_3_diff1": 0.6455323594330413, + "nauc_recall_at_3_max": 0.35678163441569755, + "nauc_recall_at_3_std": -0.30135216051748465, + "nauc_recall_at_5_diff1": 0.6284727888552449, + "nauc_recall_at_5_max": 0.35903576372560486, + "nauc_recall_at_5_std": -0.3079306794050015, + "ndcg_at_1": 0.54297, + "ndcg_at_10": 0.6669, + "ndcg_at_100": 0.69433, + "ndcg_at_1000": 0.70217, + "ndcg_at_20": 0.67905, + "ndcg_at_3": 0.62933, + "ndcg_at_5": 0.64897, + "precision_at_1": 0.54297, + "precision_at_10": 0.07921, + "precision_at_100": 0.00922, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04199, + "precision_at_3": 0.22975, + "precision_at_5": 0.14738, + "recall_at_1": 0.54297, + "recall_at_10": 0.79212, + "recall_at_100": 0.92205, + "recall_at_1000": 0.98441, + "recall_at_20": 0.83987, + "recall_at_3": 0.68924, + "recall_at_5": 0.73691 + } + ] + }, + "task_name": "CodeFeedbackMT" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CodeFeedbackST.json b/results/voyageai__voyage-3/no_revision_available/CodeFeedbackST.json new file mode 100644 index 000000000..ea01f95a5 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CodeFeedbackST.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "d213819e87aab9010628da8b73ab4eb337c89340", + "evaluation_time": 1344.3807997703552, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.83024, + "map_at_1": 0.69575, + "map_at_10": 0.79117, + "map_at_100": 0.79304, + "map_at_1000": 0.79309, + "map_at_20": 0.79242, + "map_at_3": 0.7771, + "map_at_5": 0.78673, + "mrr_at_1": 0.6942758576630678, + "mrr_at_10": 0.7904996531929027, + "mrr_at_100": 0.7923726911201066, + "mrr_at_1000": 0.7924220503403014, + "mrr_at_20": 0.7917565959334859, + "mrr_at_3": 0.7764060137566692, + "mrr_at_5": 0.7860143955365253, + "nauc_map_at_1000_diff1": 0.7776623045086383, + "nauc_map_at_1000_max": 0.21385040027084845, + "nauc_map_at_1000_std": -0.41690466439228013, + "nauc_map_at_100_diff1": 0.7776400818155216, + "nauc_map_at_100_max": 0.21385618635918036, + "nauc_map_at_100_std": -0.416942207489164, + "nauc_map_at_10_diff1": 0.7764918998316739, + "nauc_map_at_10_max": 0.21479417958796393, + "nauc_map_at_10_std": -0.4197177075581615, + "nauc_map_at_1_diff1": 0.8053343959063866, + "nauc_map_at_1_max": 0.19551406345540792, + "nauc_map_at_1_std": -0.36423944052840257, + "nauc_map_at_20_diff1": 0.7773193107145014, + "nauc_map_at_20_max": 0.21413961384129077, + "nauc_map_at_20_std": -0.4180421604514194, + "nauc_map_at_3_diff1": 0.7747386412002722, + "nauc_map_at_3_max": 0.21591056961649938, + "nauc_map_at_3_std": -0.42522401340436405, + "nauc_map_at_5_diff1": 0.7740597196301869, + "nauc_map_at_5_max": 0.21570916894101427, + "nauc_map_at_5_std": -0.42494733017640646, + "nauc_mrr_at_1000_diff1": 0.7796550783955118, + "nauc_mrr_at_1000_max": 0.21433931076730953, + "nauc_mrr_at_1000_std": -0.41603899065825717, + "nauc_mrr_at_100_diff1": 0.7796323576273196, + "nauc_mrr_at_100_max": 0.2143449666810263, + "nauc_mrr_at_100_std": -0.4160766435280458, + "nauc_mrr_at_10_diff1": 0.7784668062177206, + "nauc_mrr_at_10_max": 0.2152776634277353, + "nauc_mrr_at_10_std": -0.4188813023239788, + "nauc_mrr_at_1_diff1": 0.8075764286979211, + "nauc_mrr_at_1_max": 0.19571820322597774, + "nauc_mrr_at_1_std": -0.363340418562916, + "nauc_mrr_at_20_diff1": 0.7793080348620459, + "nauc_mrr_at_20_max": 0.21462087955006973, + "nauc_mrr_at_20_std": -0.41719569493558445, + "nauc_mrr_at_3_diff1": 0.7767338719002203, + "nauc_mrr_at_3_max": 0.2162675766492418, + "nauc_mrr_at_3_std": -0.42430859925594727, + "nauc_mrr_at_5_diff1": 0.7760548729773376, + "nauc_mrr_at_5_max": 0.2164863613421151, + "nauc_mrr_at_5_std": -0.4238427132671729, + "nauc_ndcg_at_1000_diff1": 0.7731046498790302, + "nauc_ndcg_at_1000_max": 0.2179204216232101, + "nauc_ndcg_at_1000_std": -0.42322490001678836, + "nauc_ndcg_at_100_diff1": 0.7724132887836942, + "nauc_ndcg_at_100_max": 0.21847900959342004, + "nauc_ndcg_at_100_std": -0.42366459203023804, + "nauc_ndcg_at_10_diff1": 0.7664146767625929, + "nauc_ndcg_at_10_max": 0.22320634257873917, + "nauc_ndcg_at_10_std": -0.4397032726675688, + "nauc_ndcg_at_1_diff1": 0.8053343959063866, + "nauc_ndcg_at_1_max": 0.19551406345540792, + "nauc_ndcg_at_1_std": -0.36423944052840257, + "nauc_ndcg_at_20_diff1": 0.769831814323581, + "nauc_ndcg_at_20_max": 0.22055120448567223, + "nauc_ndcg_at_20_std": -0.43278441772742365, + "nauc_ndcg_at_3_diff1": 0.7625861168141931, + "nauc_ndcg_at_3_max": 0.22494827733989517, + "nauc_ndcg_at_3_std": -0.45142731527666, + "nauc_ndcg_at_5_diff1": 0.7603310016928783, + "nauc_ndcg_at_5_max": 0.2254290213391526, + "nauc_ndcg_at_5_std": -0.45333538655954164, + "nauc_precision_at_1000_diff1": 0.5939922251007537, + "nauc_precision_at_1000_max": 0.569758502814353, + "nauc_precision_at_1000_std": 0.22596024043553337, + "nauc_precision_at_100_diff1": 0.6696076962247763, + "nauc_precision_at_100_max": 0.34752252773423276, + "nauc_precision_at_100_std": -0.3385554247738282, + "nauc_precision_at_10_diff1": 0.669680492935175, + "nauc_precision_at_10_max": 0.3082446147290942, + "nauc_precision_at_10_std": -0.6304802261801988, + "nauc_precision_at_1_diff1": 0.8053343959063866, + "nauc_precision_at_1_max": 0.19551406345540792, + "nauc_precision_at_1_std": -0.36423944052840257, + "nauc_precision_at_20_diff1": 0.6782428931902372, + "nauc_precision_at_20_max": 0.30727786656425493, + "nauc_precision_at_20_std": -0.6106167590046937, + "nauc_precision_at_3_diff1": 0.7060374963980982, + "nauc_precision_at_3_max": 0.26750926346616666, + "nauc_precision_at_3_std": -0.5744286369454905, + "nauc_precision_at_5_diff1": 0.6707839293820999, + "nauc_precision_at_5_max": 0.2895497053261246, + "nauc_precision_at_5_std": -0.6400092381083483, + "nauc_recall_at_1000_diff1": 0.5939922251006744, + "nauc_recall_at_1000_max": 0.5697585028143812, + "nauc_recall_at_1000_std": 0.22596024043562243, + "nauc_recall_at_100_diff1": 0.669607696224792, + "nauc_recall_at_100_max": 0.3475225277342409, + "nauc_recall_at_100_std": -0.33855542477379547, + "nauc_recall_at_10_diff1": 0.6696804929351756, + "nauc_recall_at_10_max": 0.30824461472909687, + "nauc_recall_at_10_std": -0.6304802261801925, + "nauc_recall_at_1_diff1": 0.8053343959063866, + "nauc_recall_at_1_max": 0.19551406345540792, + "nauc_recall_at_1_std": -0.36423944052840257, + "nauc_recall_at_20_diff1": 0.6782428931902414, + "nauc_recall_at_20_max": 0.3072778665642616, + "nauc_recall_at_20_std": -0.6106167590046883, + "nauc_recall_at_3_diff1": 0.7060374963980971, + "nauc_recall_at_3_max": 0.2675092634661662, + "nauc_recall_at_3_std": -0.5744286369454896, + "nauc_recall_at_5_diff1": 0.670783929382097, + "nauc_recall_at_5_max": 0.2895497053261264, + "nauc_recall_at_5_std": -0.6400092381083443, + "ndcg_at_1": 0.69575, + "ndcg_at_10": 0.83024, + "ndcg_at_100": 0.83899, + "ndcg_at_1000": 0.84015, + "ndcg_at_20": 0.83475, + "ndcg_at_3": 0.80247, + "ndcg_at_5": 0.81969, + "precision_at_1": 0.69575, + "precision_at_10": 0.09492, + "precision_at_100": 0.00989, + "precision_at_1000": 0.001, + "precision_at_20": 0.04835, + "precision_at_3": 0.29185, + "precision_at_5": 0.18341, + "recall_at_1": 0.69575, + "recall_at_10": 0.94924, + "recall_at_100": 0.9893, + "recall_at_1000": 0.99818, + "recall_at_20": 0.96691, + "recall_at_3": 0.87555, + "recall_at_5": 0.91704 + } + ] + }, + "task_name": "CodeFeedbackST" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CodeSearchNetCCRetrieval.json b/results/voyageai__voyage-3/no_revision_available/CodeSearchNetCCRetrieval.json new file mode 100644 index 000000000..58472cdd8 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CodeSearchNetCCRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "6e1effa2c03723c5fde48ee912b5ee08d4f211e8", + "evaluation_time": 4784.548362970352, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "python", + "languages": [ + "python-Code" + ], + "main_score": 0.798, + "map_at_1": 0.67241, + "map_at_10": 0.76055, + "map_at_100": 0.76338, + "map_at_1000": 0.76347, + "map_at_20": 0.76248, + "map_at_3": 0.74631, + "map_at_5": 0.7555, + "mrr_at_1": 0.6725432363587612, + "mrr_at_10": 0.7606110759985338, + "mrr_at_100": 0.76343349655356, + "mrr_at_1000": 0.7635235209997826, + "mrr_at_20": 0.7625285539477877, + "mrr_at_3": 0.7463690396389201, + "mrr_at_5": 0.7555525763060358, + "nauc_map_at_1000_diff1": 0.6944916346007945, + "nauc_map_at_1000_max": 0.4870205398255278, + "nauc_map_at_1000_std": -0.11351002025923429, + "nauc_map_at_100_diff1": 0.6944735519378341, + "nauc_map_at_100_max": 0.48710756101550173, + "nauc_map_at_100_std": -0.11343663401418176, + "nauc_map_at_10_diff1": 0.6935672759073939, + "nauc_map_at_10_max": 0.48785405562840306, + "nauc_map_at_10_std": -0.11485271267364111, + "nauc_map_at_1_diff1": 0.712995135428315, + "nauc_map_at_1_max": 0.44766113957896786, + "nauc_map_at_1_std": -0.11694758688822872, + "nauc_map_at_20_diff1": 0.6941828694885165, + "nauc_map_at_20_max": 0.4875750341986813, + "nauc_map_at_20_std": -0.11364496571491901, + "nauc_map_at_3_diff1": 0.6925892268094626, + "nauc_map_at_3_max": 0.4889777230685109, + "nauc_map_at_3_std": -0.11659955393050016, + "nauc_map_at_5_diff1": 0.6936096641271288, + "nauc_map_at_5_max": 0.48989436039436046, + "nauc_map_at_5_std": -0.11577555661125223, + "nauc_mrr_at_1000_diff1": 0.6942867155539255, + "nauc_mrr_at_1000_max": 0.48699701395501277, + "nauc_mrr_at_1000_std": -0.11340493848949658, + "nauc_mrr_at_100_diff1": 0.6942687085611581, + "nauc_mrr_at_100_max": 0.48708402465682304, + "nauc_mrr_at_100_std": -0.11333164126907146, + "nauc_mrr_at_10_diff1": 0.6933703308916994, + "nauc_mrr_at_10_max": 0.48782289388293965, + "nauc_mrr_at_10_std": -0.11477375514505538, + "nauc_mrr_at_1_diff1": 0.7126995615963443, + "nauc_mrr_at_1_max": 0.4476888292280168, + "nauc_mrr_at_1_std": -0.1166560112276484, + "nauc_mrr_at_20_diff1": 0.6939836089112159, + "nauc_mrr_at_20_max": 0.4875595906732648, + "nauc_mrr_at_20_std": -0.11352315070650927, + "nauc_mrr_at_3_diff1": 0.6923824710864314, + "nauc_mrr_at_3_max": 0.48897330011701334, + "nauc_mrr_at_3_std": -0.1165024581665663, + "nauc_mrr_at_5_diff1": 0.6934125747554679, + "nauc_mrr_at_5_max": 0.4898760870720158, + "nauc_mrr_at_5_std": -0.11566752205697262, + "nauc_ndcg_at_1000_diff1": 0.6916678048384453, + "nauc_ndcg_at_1000_max": 0.49620006458777205, + "nauc_ndcg_at_1000_std": -0.10738746083777574, + "nauc_ndcg_at_100_diff1": 0.6910221007473211, + "nauc_ndcg_at_100_max": 0.4988849920626929, + "nauc_ndcg_at_100_std": -0.10495291366167504, + "nauc_ndcg_at_10_diff1": 0.68639695032148, + "nauc_ndcg_at_10_max": 0.503449745844292, + "nauc_ndcg_at_10_std": -0.11112621917144268, + "nauc_ndcg_at_1_diff1": 0.712995135428315, + "nauc_ndcg_at_1_max": 0.44766113957896786, + "nauc_ndcg_at_1_std": -0.11694758688822872, + "nauc_ndcg_at_20_diff1": 0.6886076204860218, + "nauc_ndcg_at_20_max": 0.5028601804327565, + "nauc_ndcg_at_20_std": -0.10595051229239155, + "nauc_ndcg_at_3_diff1": 0.6852040536518835, + "nauc_ndcg_at_3_max": 0.5049799528956791, + "nauc_ndcg_at_3_std": -0.11560039193354153, + "nauc_ndcg_at_5_diff1": 0.6867304314581527, + "nauc_ndcg_at_5_max": 0.5080125305814686, + "nauc_ndcg_at_5_std": -0.11380567625269568, + "nauc_precision_at_1000_diff1": 0.7056728935148587, + "nauc_precision_at_1000_max": 0.7557038050853563, + "nauc_precision_at_1000_std": 0.34716069321571114, + "nauc_precision_at_100_diff1": 0.6605297368835459, + "nauc_precision_at_100_max": 0.6982508497047939, + "nauc_precision_at_100_std": 0.13495310796487917, + "nauc_precision_at_10_diff1": 0.6383909988061885, + "nauc_precision_at_10_max": 0.6074079131948248, + "nauc_precision_at_10_std": -0.08168531737507122, + "nauc_precision_at_1_diff1": 0.712995135428315, + "nauc_precision_at_1_max": 0.44766113957896786, + "nauc_precision_at_1_std": -0.11694758688822872, + "nauc_precision_at_20_diff1": 0.6404772580227832, + "nauc_precision_at_20_max": 0.6482961081311777, + "nauc_precision_at_20_std": -0.010676250590994963, + "nauc_precision_at_3_diff1": 0.6553774533027522, + "nauc_precision_at_3_max": 0.570117837039712, + "nauc_precision_at_3_std": -0.11114106284233406, + "nauc_precision_at_5_diff1": 0.652746794538016, + "nauc_precision_at_5_max": 0.6002323497296262, + "nauc_precision_at_5_std": -0.10281875029760373, + "nauc_recall_at_1000_diff1": 0.7056728935148543, + "nauc_recall_at_1000_max": 0.7557038050853054, + "nauc_recall_at_1000_std": 0.3471606932157271, + "nauc_recall_at_100_diff1": 0.6605297368835513, + "nauc_recall_at_100_max": 0.6982508497047836, + "nauc_recall_at_100_std": 0.13495310796487786, + "nauc_recall_at_10_diff1": 0.6383909988061875, + "nauc_recall_at_10_max": 0.6074079131948261, + "nauc_recall_at_10_std": -0.08168531737506997, + "nauc_recall_at_1_diff1": 0.712995135428315, + "nauc_recall_at_1_max": 0.44766113957896786, + "nauc_recall_at_1_std": -0.11694758688822872, + "nauc_recall_at_20_diff1": 0.6404772580227842, + "nauc_recall_at_20_max": 0.6482961081311792, + "nauc_recall_at_20_std": -0.010676250590993822, + "nauc_recall_at_3_diff1": 0.6553774533027508, + "nauc_recall_at_3_max": 0.5701178370397109, + "nauc_recall_at_3_std": -0.11114106284233452, + "nauc_recall_at_5_diff1": 0.6527467945380182, + "nauc_recall_at_5_max": 0.6002323497296274, + "nauc_recall_at_5_std": -0.10281875029760265, + "ndcg_at_1": 0.67241, + "ndcg_at_10": 0.798, + "ndcg_at_100": 0.81133, + "ndcg_at_1000": 0.81375, + "ndcg_at_20": 0.80495, + "ndcg_at_3": 0.76945, + "ndcg_at_5": 0.78592, + "precision_at_1": 0.67241, + "precision_at_10": 0.09129, + "precision_at_100": 0.00974, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04701, + "precision_at_3": 0.2787, + "precision_at_5": 0.17518, + "recall_at_1": 0.67241, + "recall_at_10": 0.91292, + "recall_at_100": 0.97433, + "recall_at_1000": 0.99343, + "recall_at_20": 0.94027, + "recall_at_3": 0.8361, + "recall_at_5": 0.87592 + }, + { + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ], + "main_score": 0.81058, + "map_at_1": 0.69037, + "map_at_10": 0.77576, + "map_at_100": 0.77835, + "map_at_1000": 0.77844, + "map_at_20": 0.7774, + "map_at_3": 0.76269, + "map_at_5": 0.77138, + "mrr_at_1": 0.6903676694013977, + "mrr_at_10": 0.7757635060506534, + "mrr_at_100": 0.7783551663123122, + "mrr_at_1000": 0.778443072798153, + "mrr_at_20": 0.7773994141561121, + "mrr_at_3": 0.7626861136432701, + "mrr_at_5": 0.7713764813126714, + "nauc_map_at_1000_diff1": 0.7042004886861938, + "nauc_map_at_1000_max": 0.46823424900450317, + "nauc_map_at_1000_std": -0.12259323066997758, + "nauc_map_at_100_diff1": 0.7041634280782116, + "nauc_map_at_100_max": 0.4683276293874495, + "nauc_map_at_100_std": -0.12245771711784639, + "nauc_map_at_10_diff1": 0.703669300854218, + "nauc_map_at_10_max": 0.47072424247301203, + "nauc_map_at_10_std": -0.12267638986094445, + "nauc_map_at_1_diff1": 0.7268604440507198, + "nauc_map_at_1_max": 0.4104612347356543, + "nauc_map_at_1_std": -0.14517597678099742, + "nauc_map_at_20_diff1": 0.7037099535441743, + "nauc_map_at_20_max": 0.4691712438798155, + "nauc_map_at_20_std": -0.12213527842185298, + "nauc_map_at_3_diff1": 0.7042210642000994, + "nauc_map_at_3_max": 0.4709958771866305, + "nauc_map_at_3_std": -0.12963287987149033, + "nauc_map_at_5_diff1": 0.7022918453865188, + "nauc_map_at_5_max": 0.4694584240176629, + "nauc_map_at_5_std": -0.12514106295935512, + "nauc_mrr_at_1000_diff1": 0.7041949779583212, + "nauc_mrr_at_1000_max": 0.46830058002629854, + "nauc_mrr_at_1000_std": -0.12258861476091411, + "nauc_mrr_at_100_diff1": 0.7041579322191078, + "nauc_mrr_at_100_max": 0.46839389639352125, + "nauc_mrr_at_100_std": -0.12245310888827152, + "nauc_mrr_at_10_diff1": 0.7036638715256678, + "nauc_mrr_at_10_max": 0.47078967056867005, + "nauc_mrr_at_10_std": -0.1226718439414119, + "nauc_mrr_at_1_diff1": 0.7268604440507198, + "nauc_mrr_at_1_max": 0.4104612347356543, + "nauc_mrr_at_1_std": -0.14517597678099742, + "nauc_mrr_at_20_diff1": 0.7037044796045289, + "nauc_mrr_at_20_max": 0.4692371985286129, + "nauc_mrr_at_20_std": -0.12213068917223761, + "nauc_mrr_at_3_diff1": 0.7042210642000994, + "nauc_mrr_at_3_max": 0.4709958771866305, + "nauc_mrr_at_3_std": -0.12963287987149033, + "nauc_mrr_at_5_diff1": 0.7022918453865188, + "nauc_mrr_at_5_max": 0.4694584240176629, + "nauc_mrr_at_5_std": -0.12514106295935512, + "nauc_ndcg_at_1000_diff1": 0.6996868715025774, + "nauc_ndcg_at_1000_max": 0.48038391907385547, + "nauc_ndcg_at_1000_std": -0.11181362679743506, + "nauc_ndcg_at_100_diff1": 0.6982476216309004, + "nauc_ndcg_at_100_max": 0.4834639684795564, + "nauc_ndcg_at_100_std": -0.10664299062480084, + "nauc_ndcg_at_10_diff1": 0.6945542722738761, + "nauc_ndcg_at_10_max": 0.4970209624484458, + "nauc_ndcg_at_10_std": -0.10420162996682648, + "nauc_ndcg_at_1_diff1": 0.7268604440507198, + "nauc_ndcg_at_1_max": 0.4104612347356543, + "nauc_ndcg_at_1_std": -0.14517597678099742, + "nauc_ndcg_at_20_diff1": 0.6945536673818254, + "nauc_ndcg_at_20_max": 0.4912810011400926, + "nauc_ndcg_at_20_std": -0.10154749824853355, + "nauc_ndcg_at_3_diff1": 0.6955824170175968, + "nauc_ndcg_at_3_max": 0.49357335647375455, + "nauc_ndcg_at_3_std": -0.12175428729377595, + "nauc_ndcg_at_5_diff1": 0.6910407140647552, + "nauc_ndcg_at_5_max": 0.4921156072195089, + "nauc_ndcg_at_5_std": -0.11205694553997607, + "nauc_precision_at_1000_diff1": 0.6817422973797523, + "nauc_precision_at_1000_max": 0.7104370824351197, + "nauc_precision_at_1000_std": 0.1497998684352135, + "nauc_precision_at_100_diff1": 0.616005703114966, + "nauc_precision_at_100_max": 0.6987939375240783, + "nauc_precision_at_100_std": 0.23437838767420707, + "nauc_precision_at_10_diff1": 0.6344694339230965, + "nauc_precision_at_10_max": 0.679978557342506, + "nauc_precision_at_10_std": 0.034696955585544295, + "nauc_precision_at_1_diff1": 0.7268604440507198, + "nauc_precision_at_1_max": 0.4104612347356543, + "nauc_precision_at_1_std": -0.14517597678099742, + "nauc_precision_at_20_diff1": 0.6124716236017355, + "nauc_precision_at_20_max": 0.6875201520706243, + "nauc_precision_at_20_std": 0.1140844984588217, + "nauc_precision_at_3_diff1": 0.6601899315546508, + "nauc_precision_at_3_max": 0.5858270973130154, + "nauc_precision_at_3_std": -0.08855605006879981, + "nauc_precision_at_5_diff1": 0.6326598799046783, + "nauc_precision_at_5_max": 0.6064081028394852, + "nauc_precision_at_5_std": -0.04172771410675072, + "nauc_recall_at_1000_diff1": 0.6817422973797576, + "nauc_recall_at_1000_max": 0.7104370824351167, + "nauc_recall_at_1000_std": 0.14979986843523688, + "nauc_recall_at_100_diff1": 0.6160057031149636, + "nauc_recall_at_100_max": 0.6987939375240682, + "nauc_recall_at_100_std": 0.23437838767420327, + "nauc_recall_at_10_diff1": 0.6344694339230996, + "nauc_recall_at_10_max": 0.6799785573425072, + "nauc_recall_at_10_std": 0.034696955585548236, + "nauc_recall_at_1_diff1": 0.7268604440507198, + "nauc_recall_at_1_max": 0.4104612347356543, + "nauc_recall_at_1_std": -0.14517597678099742, + "nauc_recall_at_20_diff1": 0.6124716236017379, + "nauc_recall_at_20_max": 0.6875201520706272, + "nauc_recall_at_20_std": 0.11408449845882308, + "nauc_recall_at_3_diff1": 0.6601899315546512, + "nauc_recall_at_3_max": 0.5858270973130163, + "nauc_recall_at_3_std": -0.08855605006879981, + "nauc_recall_at_5_diff1": 0.6326598799046776, + "nauc_recall_at_5_max": 0.6064081028394869, + "nauc_recall_at_5_std": -0.041727714106749324, + "ndcg_at_1": 0.69037, + "ndcg_at_10": 0.81058, + "ndcg_at_100": 0.82331, + "ndcg_at_1000": 0.82564, + "ndcg_at_20": 0.81641, + "ndcg_at_3": 0.78445, + "ndcg_at_5": 0.79997, + "precision_at_1": 0.69037, + "precision_at_10": 0.0917, + "precision_at_100": 0.00977, + "precision_at_1000": 0.001, + "precision_at_20": 0.04699, + "precision_at_3": 0.28229, + "precision_at_5": 0.17685, + "recall_at_1": 0.69037, + "recall_at_10": 0.91705, + "recall_at_100": 0.97691, + "recall_at_1000": 0.99514, + "recall_at_20": 0.93984, + "recall_at_3": 0.84686, + "recall_at_5": 0.88423 + }, + { + "hf_subset": "go", + "languages": [ + "go-Code" + ], + "main_score": 0.75776, + "map_at_1": 0.63088, + "map_at_10": 0.71851, + "map_at_100": 0.72208, + "map_at_1000": 0.72223, + "map_at_20": 0.72069, + "map_at_3": 0.70169, + "map_at_5": 0.71241, + "mrr_at_1": 0.6311253385865551, + "mrr_at_10": 0.718621771946073, + "mrr_at_100": 0.7221837875791267, + "mrr_at_1000": 0.7223387034016141, + "mrr_at_20": 0.7207973426879688, + "mrr_at_3": 0.7018181071985528, + "mrr_at_5": 0.7124620372650365, + "nauc_map_at_1000_diff1": 0.6456271156937657, + "nauc_map_at_1000_max": 0.45088862754211045, + "nauc_map_at_1000_std": -0.155676715822139, + "nauc_map_at_100_diff1": 0.6455805950130173, + "nauc_map_at_100_max": 0.4508501409581644, + "nauc_map_at_100_std": -0.15573941572236327, + "nauc_map_at_10_diff1": 0.645291783977216, + "nauc_map_at_10_max": 0.4513768340719191, + "nauc_map_at_10_std": -0.15691432669297392, + "nauc_map_at_1_diff1": 0.6659173111820048, + "nauc_map_at_1_max": 0.4341833217616385, + "nauc_map_at_1_std": -0.1373854722684089, + "nauc_map_at_20_diff1": 0.6451841804575909, + "nauc_map_at_20_max": 0.45102200645220053, + "nauc_map_at_20_std": -0.15600206922344606, + "nauc_map_at_3_diff1": 0.6423349529448482, + "nauc_map_at_3_max": 0.44845043850901334, + "nauc_map_at_3_std": -0.15780238826169088, + "nauc_map_at_5_diff1": 0.6448183815876021, + "nauc_map_at_5_max": 0.45263127138529186, + "nauc_map_at_5_std": -0.1598430844994673, + "nauc_mrr_at_1000_diff1": 0.6452921570604533, + "nauc_mrr_at_1000_max": 0.4505917640779232, + "nauc_mrr_at_1000_std": -0.15531484493216974, + "nauc_mrr_at_100_diff1": 0.645245820273172, + "nauc_mrr_at_100_max": 0.4505536692556074, + "nauc_mrr_at_100_std": -0.15537761283031745, + "nauc_mrr_at_10_diff1": 0.6449659502758599, + "nauc_mrr_at_10_max": 0.45108261566852625, + "nauc_mrr_at_10_std": -0.15656614044161815, + "nauc_mrr_at_1_diff1": 0.6653947207626045, + "nauc_mrr_at_1_max": 0.4336307302767222, + "nauc_mrr_at_1_std": -0.13676794144341486, + "nauc_mrr_at_20_diff1": 0.6448513012531469, + "nauc_mrr_at_20_max": 0.45072757915708417, + "nauc_mrr_at_20_std": -0.1556423618721711, + "nauc_mrr_at_3_diff1": 0.6420416241488471, + "nauc_mrr_at_3_max": 0.44827387581125583, + "nauc_mrr_at_3_std": -0.157469220409675, + "nauc_mrr_at_5_diff1": 0.6445319860804638, + "nauc_mrr_at_5_max": 0.4524529445989842, + "nauc_mrr_at_5_std": -0.15966460796139567, + "nauc_ndcg_at_1000_diff1": 0.6421473034308012, + "nauc_ndcg_at_1000_max": 0.4554018983886954, + "nauc_ndcg_at_1000_std": -0.1559007102469374, + "nauc_ndcg_at_100_diff1": 0.6408293336048709, + "nauc_ndcg_at_100_max": 0.455326508003864, + "nauc_ndcg_at_100_std": -0.15623695118338465, + "nauc_ndcg_at_10_diff1": 0.6389026793025616, + "nauc_ndcg_at_10_max": 0.4578600723653024, + "nauc_ndcg_at_10_std": -0.1625832644618312, + "nauc_ndcg_at_1_diff1": 0.6659173111820048, + "nauc_ndcg_at_1_max": 0.4341833217616385, + "nauc_ndcg_at_1_std": -0.1373854722684089, + "nauc_ndcg_at_20_diff1": 0.6381715507471929, + "nauc_ndcg_at_20_max": 0.45663828833439424, + "nauc_ndcg_at_20_std": -0.15884479519112304, + "nauc_ndcg_at_3_diff1": 0.6335257496919433, + "nauc_ndcg_at_3_max": 0.4521885893360026, + "nauc_ndcg_at_3_std": -0.1652822819033794, + "nauc_ndcg_at_5_diff1": 0.6380125573519403, + "nauc_ndcg_at_5_max": 0.4609777757089876, + "nauc_ndcg_at_5_std": -0.16975297234970457, + "nauc_precision_at_1000_diff1": 0.5977425801910219, + "nauc_precision_at_1000_max": 0.7227252270709481, + "nauc_precision_at_1000_std": 0.12753697406512687, + "nauc_precision_at_100_diff1": 0.5846421166687563, + "nauc_precision_at_100_max": 0.5116549312017075, + "nauc_precision_at_100_std": -0.10302770797636711, + "nauc_precision_at_10_diff1": 0.6043030419847906, + "nauc_precision_at_10_max": 0.4946025837429335, + "nauc_precision_at_10_std": -0.19140935519721827, + "nauc_precision_at_1_diff1": 0.6659173111820048, + "nauc_precision_at_1_max": 0.4341833217616385, + "nauc_precision_at_1_std": -0.1373854722684089, + "nauc_precision_at_20_diff1": 0.586840591486799, + "nauc_precision_at_20_max": 0.4953606556157186, + "nauc_precision_at_20_std": -0.1669870559062597, + "nauc_precision_at_3_diff1": 0.601034763907146, + "nauc_precision_at_3_max": 0.4653319244245944, + "nauc_precision_at_3_std": -0.19281093517283046, + "nauc_precision_at_5_diff1": 0.6090574563512017, + "nauc_precision_at_5_max": 0.49950606252854807, + "nauc_precision_at_5_std": -0.21459252871221965, + "nauc_recall_at_1000_diff1": 0.5977425801910333, + "nauc_recall_at_1000_max": 0.7227252270709641, + "nauc_recall_at_1000_std": 0.12753697406513756, + "nauc_recall_at_100_diff1": 0.5846421166687518, + "nauc_recall_at_100_max": 0.5116549312016969, + "nauc_recall_at_100_std": -0.10302770797636562, + "nauc_recall_at_10_diff1": 0.6043030419847915, + "nauc_recall_at_10_max": 0.49460258374293237, + "nauc_recall_at_10_std": -0.19140935519721794, + "nauc_recall_at_1_diff1": 0.6659173111820048, + "nauc_recall_at_1_max": 0.4341833217616385, + "nauc_recall_at_1_std": -0.1373854722684089, + "nauc_recall_at_20_diff1": 0.5868405914867991, + "nauc_recall_at_20_max": 0.49536065561571957, + "nauc_recall_at_20_std": -0.16698705590625929, + "nauc_recall_at_3_diff1": 0.6010347639071463, + "nauc_recall_at_3_max": 0.4653319244245934, + "nauc_recall_at_3_std": -0.19281093517283265, + "nauc_recall_at_5_diff1": 0.6090574563512018, + "nauc_recall_at_5_max": 0.4995060625285478, + "nauc_recall_at_5_std": -0.21459252871221543, + "ndcg_at_1": 0.63088, + "ndcg_at_10": 0.75776, + "ndcg_at_100": 0.77526, + "ndcg_at_1000": 0.7793, + "ndcg_at_20": 0.76557, + "ndcg_at_3": 0.72381, + "ndcg_at_5": 0.7431, + "precision_at_1": 0.63088, + "precision_at_10": 0.08793, + "precision_at_100": 0.00962, + "precision_at_1000": 0.00099, + "precision_at_20": 0.0455, + "precision_at_3": 0.2625, + "precision_at_5": 0.16686, + "recall_at_1": 0.63088, + "recall_at_10": 0.87934, + "recall_at_100": 0.96159, + "recall_at_1000": 0.99323, + "recall_at_20": 0.91, + "recall_at_3": 0.78749, + "recall_at_5": 0.83428 + }, + { + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ], + "main_score": 0.79512, + "map_at_1": 0.67962, + "map_at_10": 0.75761, + "map_at_100": 0.76014, + "map_at_1000": 0.76026, + "map_at_20": 0.75906, + "map_at_3": 0.74108, + "map_at_5": 0.75032, + "mrr_at_1": 0.6796193497224425, + "mrr_at_10": 0.757813149050263, + "mrr_at_100": 0.760342440919257, + "mrr_at_1000": 0.7604570343951252, + "mrr_at_20": 0.7592561407845735, + "mrr_at_3": 0.7412106793550098, + "mrr_at_5": 0.7506476341527898, + "nauc_map_at_1000_diff1": 0.7205041125532301, + "nauc_map_at_1000_max": 0.5337551222221237, + "nauc_map_at_1000_std": -0.15179484105391383, + "nauc_map_at_100_diff1": 0.7204936915194204, + "nauc_map_at_100_max": 0.5338250417572229, + "nauc_map_at_100_std": -0.15169483911206375, + "nauc_map_at_10_diff1": 0.7182698517884425, + "nauc_map_at_10_max": 0.5334876028185493, + "nauc_map_at_10_std": -0.15096257613543615, + "nauc_map_at_1_diff1": 0.7417636870086498, + "nauc_map_at_1_max": 0.5152361166741234, + "nauc_map_at_1_std": -0.143467642736416, + "nauc_map_at_20_diff1": 0.7203511303874609, + "nauc_map_at_20_max": 0.53450842109563, + "nauc_map_at_20_std": -0.15174952296208583, + "nauc_map_at_3_diff1": 0.7170745379047607, + "nauc_map_at_3_max": 0.5353330346614941, + "nauc_map_at_3_std": -0.152667270080009, + "nauc_map_at_5_diff1": 0.7183357085697772, + "nauc_map_at_5_max": 0.5337365147911263, + "nauc_map_at_5_std": -0.15646513831569409, + "nauc_mrr_at_1000_diff1": 0.7202760310775567, + "nauc_mrr_at_1000_max": 0.5362563737085365, + "nauc_mrr_at_1000_std": -0.15132594039580743, + "nauc_mrr_at_100_diff1": 0.720265727100864, + "nauc_mrr_at_100_max": 0.5363249771019998, + "nauc_mrr_at_100_std": -0.15122612165124702, + "nauc_mrr_at_10_diff1": 0.7180423786702759, + "nauc_mrr_at_10_max": 0.5359582347298918, + "nauc_mrr_at_10_std": -0.1504960330802269, + "nauc_mrr_at_1_diff1": 0.7417636870086498, + "nauc_mrr_at_1_max": 0.5191209665329816, + "nauc_mrr_at_1_std": -0.14289284642831523, + "nauc_mrr_at_20_diff1": 0.7201236098337845, + "nauc_mrr_at_20_max": 0.5369967122918928, + "nauc_mrr_at_20_std": -0.151280356130205, + "nauc_mrr_at_3_diff1": 0.7169554368042463, + "nauc_mrr_at_3_max": 0.5374533828298951, + "nauc_mrr_at_3_std": -0.15259769042134338, + "nauc_mrr_at_5_diff1": 0.7179668805336076, + "nauc_mrr_at_5_max": 0.5361988728683226, + "nauc_mrr_at_5_std": -0.15504551909459952, + "nauc_ndcg_at_1000_diff1": 0.7186793890459555, + "nauc_ndcg_at_1000_max": 0.5378795592010746, + "nauc_ndcg_at_1000_std": -0.15267210049181826, + "nauc_ndcg_at_100_diff1": 0.7184818373869992, + "nauc_ndcg_at_100_max": 0.5390372406911389, + "nauc_ndcg_at_100_std": -0.1511082175526577, + "nauc_ndcg_at_10_diff1": 0.7095433838787453, + "nauc_ndcg_at_10_max": 0.5401566811793684, + "nauc_ndcg_at_10_std": -0.14897868957395116, + "nauc_ndcg_at_1_diff1": 0.7417636870086498, + "nauc_ndcg_at_1_max": 0.5152361166741234, + "nauc_ndcg_at_1_std": -0.143467642736416, + "nauc_ndcg_at_20_diff1": 0.7177560377421203, + "nauc_ndcg_at_20_max": 0.5443798772778566, + "nauc_ndcg_at_20_std": -0.1521165225315314, + "nauc_ndcg_at_3_diff1": 0.7085169758019074, + "nauc_ndcg_at_3_max": 0.5436328778592758, + "nauc_ndcg_at_3_std": -0.15712774449120478, + "nauc_ndcg_at_5_diff1": 0.7100362832502062, + "nauc_ndcg_at_5_max": 0.5407669251365145, + "nauc_ndcg_at_5_std": -0.16514434909187362, + "nauc_precision_at_1000_diff1": 0.9128617085600557, + "nauc_precision_at_1000_max": 0.7642792961677345, + "nauc_precision_at_1000_std": -0.0730637817997648, + "nauc_precision_at_100_diff1": 0.7298041592831268, + "nauc_precision_at_100_max": 0.607172536796614, + "nauc_precision_at_100_std": -0.0848283039351695, + "nauc_precision_at_10_diff1": 0.6496505665384006, + "nauc_precision_at_10_max": 0.584076914910153, + "nauc_precision_at_10_std": -0.12057892476603654, + "nauc_precision_at_1_diff1": 0.7417636870086498, + "nauc_precision_at_1_max": 0.5152361166741234, + "nauc_precision_at_1_std": -0.143467642736416, + "nauc_precision_at_20_diff1": 0.7146080815699437, + "nauc_precision_at_20_max": 0.6388121102864954, + "nauc_precision_at_20_std": -0.14371308875373734, + "nauc_precision_at_3_diff1": 0.6764159795332267, + "nauc_precision_at_3_max": 0.575371061787295, + "nauc_precision_at_3_std": -0.17443557185521524, + "nauc_precision_at_5_diff1": 0.6720918609304856, + "nauc_precision_at_5_max": 0.5726163410214501, + "nauc_precision_at_5_std": -0.20782834347527765, + "nauc_recall_at_1000_diff1": 0.9128617085600882, + "nauc_recall_at_1000_max": 0.7642792961677456, + "nauc_recall_at_1000_std": -0.07306378179979198, + "nauc_recall_at_100_diff1": 0.7298041592831347, + "nauc_recall_at_100_max": 0.6071725367966184, + "nauc_recall_at_100_std": -0.08482830393515768, + "nauc_recall_at_10_diff1": 0.6496505665383987, + "nauc_recall_at_10_max": 0.584076914910151, + "nauc_recall_at_10_std": -0.12057892476603893, + "nauc_recall_at_1_diff1": 0.7417636870086498, + "nauc_recall_at_1_max": 0.5152361166741234, + "nauc_recall_at_1_std": -0.143467642736416, + "nauc_recall_at_20_diff1": 0.7146080815699448, + "nauc_recall_at_20_max": 0.6388121102864951, + "nauc_recall_at_20_std": -0.1437130887537365, + "nauc_recall_at_3_diff1": 0.6764159795332266, + "nauc_recall_at_3_max": 0.5753710617872944, + "nauc_recall_at_3_std": -0.17443557185521374, + "nauc_recall_at_5_diff1": 0.6720918609304842, + "nauc_recall_at_5_max": 0.5726163410214484, + "nauc_recall_at_5_std": -0.207828343475278, + "ndcg_at_1": 0.67962, + "ndcg_at_10": 0.79512, + "ndcg_at_100": 0.80792, + "ndcg_at_1000": 0.81096, + "ndcg_at_20": 0.80022, + "ndcg_at_3": 0.76076, + "ndcg_at_5": 0.77758, + "precision_at_1": 0.67962, + "precision_at_10": 0.09128, + "precision_at_100": 0.00974, + "precision_at_1000": 0.001, + "precision_at_20": 0.04663, + "precision_at_3": 0.27254, + "precision_at_5": 0.17177, + "recall_at_1": 0.67962, + "recall_at_10": 0.91277, + "recall_at_100": 0.97383, + "recall_at_1000": 0.99762, + "recall_at_20": 0.93259, + "recall_at_3": 0.81761, + "recall_at_5": 0.85884 + }, + { + "hf_subset": "java", + "languages": [ + "java-Code" + ], + "main_score": 0.79747, + "map_at_1": 0.67887, + "map_at_10": 0.76255, + "map_at_100": 0.76511, + "map_at_1000": 0.76524, + "map_at_20": 0.76427, + "map_at_3": 0.74932, + "map_at_5": 0.75797, + "mrr_at_1": 0.6789593792788681, + "mrr_at_10": 0.7625945969442045, + "mrr_at_100": 0.7651512702005464, + "mrr_at_1000": 0.7652827234105899, + "mrr_at_20": 0.764317504866474, + "mrr_at_3": 0.7493990567472946, + "mrr_at_5": 0.7580206907043905, + "nauc_map_at_1000_diff1": 0.6864219158673435, + "nauc_map_at_1000_max": 0.504907344422531, + "nauc_map_at_1000_std": -0.19437059468670823, + "nauc_map_at_100_diff1": 0.6864218924062259, + "nauc_map_at_100_max": 0.505030495640309, + "nauc_map_at_100_std": -0.19445734435161083, + "nauc_map_at_10_diff1": 0.6856323148259689, + "nauc_map_at_10_max": 0.5063105444235586, + "nauc_map_at_10_std": -0.1936294907086918, + "nauc_map_at_1_diff1": 0.7035774784793363, + "nauc_map_at_1_max": 0.45315190797040206, + "nauc_map_at_1_std": -0.18522305898762378, + "nauc_map_at_20_diff1": 0.686164863750196, + "nauc_map_at_20_max": 0.5053944501197204, + "nauc_map_at_20_std": -0.1940935026059026, + "nauc_map_at_3_diff1": 0.684465569293455, + "nauc_map_at_3_max": 0.5068800377501743, + "nauc_map_at_3_std": -0.19485395833861172, + "nauc_map_at_5_diff1": 0.6846956792984705, + "nauc_map_at_5_max": 0.5071958407953111, + "nauc_map_at_5_std": -0.194880323642238, + "nauc_mrr_at_1000_diff1": 0.6863274949598669, + "nauc_mrr_at_1000_max": 0.5050267805117991, + "nauc_mrr_at_1000_std": -0.19466516947075846, + "nauc_mrr_at_100_diff1": 0.6863275306904171, + "nauc_mrr_at_100_max": 0.5051498697363815, + "nauc_mrr_at_100_std": -0.1947517544100277, + "nauc_mrr_at_10_diff1": 0.6855381350066083, + "nauc_mrr_at_10_max": 0.5064276492418595, + "nauc_mrr_at_10_std": -0.1939450041919192, + "nauc_mrr_at_1_diff1": 0.7033718326616138, + "nauc_mrr_at_1_max": 0.45333067098804714, + "nauc_mrr_at_1_std": -0.1855524577066343, + "nauc_mrr_at_20_diff1": 0.6860710165606818, + "nauc_mrr_at_20_max": 0.5055136304911908, + "nauc_mrr_at_20_std": -0.19438659089055144, + "nauc_mrr_at_3_diff1": 0.6843676607272918, + "nauc_mrr_at_3_max": 0.506936939220341, + "nauc_mrr_at_3_std": -0.1950981353562335, + "nauc_mrr_at_5_diff1": 0.6845885860292696, + "nauc_mrr_at_5_max": 0.5072858032701117, + "nauc_mrr_at_5_std": -0.19525348044904925, + "nauc_ndcg_at_1000_diff1": 0.6850085729456115, + "nauc_ndcg_at_1000_max": 0.5176997380520324, + "nauc_ndcg_at_1000_std": -0.1971272390955728, + "nauc_ndcg_at_100_diff1": 0.6849268342574114, + "nauc_ndcg_at_100_max": 0.5218858316196956, + "nauc_ndcg_at_100_std": -0.1991841419933876, + "nauc_ndcg_at_10_diff1": 0.6806827297442719, + "nauc_ndcg_at_10_max": 0.5278250816192698, + "nauc_ndcg_at_10_std": -0.19481640466457897, + "nauc_ndcg_at_1_diff1": 0.7035774784793363, + "nauc_ndcg_at_1_max": 0.45315190797040206, + "nauc_ndcg_at_1_std": -0.18522305898762378, + "nauc_ndcg_at_20_diff1": 0.6827802797713385, + "nauc_ndcg_at_20_max": 0.5247482268456363, + "nauc_ndcg_at_20_std": -0.19699040837915718, + "nauc_ndcg_at_3_diff1": 0.6784051060670401, + "nauc_ndcg_at_3_max": 0.5272089839672903, + "nauc_ndcg_at_3_std": -0.1980298585102133, + "nauc_ndcg_at_5_diff1": 0.6784138791849607, + "nauc_ndcg_at_5_max": 0.5290173532682292, + "nauc_ndcg_at_5_std": -0.19819070661644098, + "nauc_precision_at_1000_diff1": 0.7486763235496546, + "nauc_precision_at_1000_max": 0.7873122004385744, + "nauc_precision_at_1000_std": -0.28078309807717683, + "nauc_precision_at_100_diff1": 0.6969712406592434, + "nauc_precision_at_100_max": 0.7322835784240599, + "nauc_precision_at_100_std": -0.28336626798905784, + "nauc_precision_at_10_diff1": 0.6526681420478505, + "nauc_precision_at_10_max": 0.6611587653631131, + "nauc_precision_at_10_std": -0.19874404599419027, + "nauc_precision_at_1_diff1": 0.7035774784793363, + "nauc_precision_at_1_max": 0.45315190797040206, + "nauc_precision_at_1_std": -0.18522305898762378, + "nauc_precision_at_20_diff1": 0.6634012407151632, + "nauc_precision_at_20_max": 0.6768172892774131, + "nauc_precision_at_20_std": -0.2217825029129253, + "nauc_precision_at_3_diff1": 0.6548342535842161, + "nauc_precision_at_3_max": 0.6080476423566363, + "nauc_precision_at_3_std": -0.21044751850866933, + "nauc_precision_at_5_diff1": 0.6486691672371485, + "nauc_precision_at_5_max": 0.6347553820364135, + "nauc_precision_at_5_std": -0.21385196834259307, + "nauc_recall_at_1000_diff1": 0.7486763235496586, + "nauc_recall_at_1000_max": 0.787312200438559, + "nauc_recall_at_1000_std": -0.28078309807714774, + "nauc_recall_at_100_diff1": 0.6969712406592431, + "nauc_recall_at_100_max": 0.7322835784240567, + "nauc_recall_at_100_std": -0.2833662679890637, + "nauc_recall_at_10_diff1": 0.65266814204785, + "nauc_recall_at_10_max": 0.6611587653631126, + "nauc_recall_at_10_std": -0.19874404599418663, + "nauc_recall_at_1_diff1": 0.7035774784793363, + "nauc_recall_at_1_max": 0.45315190797040206, + "nauc_recall_at_1_std": -0.18522305898762378, + "nauc_recall_at_20_diff1": 0.6634012407151622, + "nauc_recall_at_20_max": 0.6768172892774157, + "nauc_recall_at_20_std": -0.22178250291292556, + "nauc_recall_at_3_diff1": 0.6548342535842151, + "nauc_recall_at_3_max": 0.6080476423566361, + "nauc_recall_at_3_std": -0.210447518508669, + "nauc_recall_at_5_diff1": 0.6486691672371475, + "nauc_recall_at_5_max": 0.6347553820364128, + "nauc_recall_at_5_std": -0.21385196834259515, + "ndcg_at_1": 0.67887, + "ndcg_at_10": 0.79747, + "ndcg_at_100": 0.80964, + "ndcg_at_1000": 0.81329, + "ndcg_at_20": 0.80363, + "ndcg_at_3": 0.77102, + "ndcg_at_5": 0.78652, + "precision_at_1": 0.67887, + "precision_at_10": 0.09044, + "precision_at_100": 0.00961, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04643, + "precision_at_3": 0.2778, + "precision_at_5": 0.17417, + "recall_at_1": 0.67887, + "recall_at_10": 0.90443, + "recall_at_100": 0.96084, + "recall_at_1000": 0.98996, + "recall_at_20": 0.92853, + "recall_at_3": 0.83341, + "recall_at_5": 0.87084 + }, + { + "hf_subset": "php", + "languages": [ + "php-Code" + ], + "main_score": 0.71298, + "map_at_1": 0.5735, + "map_at_10": 0.66962, + "map_at_100": 0.67385, + "map_at_1000": 0.67403, + "map_at_20": 0.67227, + "map_at_3": 0.651, + "map_at_5": 0.66287, + "mrr_at_1": 0.5737120022834309, + "mrr_at_10": 0.6697086077188115, + "mrr_at_100": 0.6739356524602215, + "mrr_at_1000": 0.674121374711382, + "mrr_at_20": 0.6723616084882621, + "mrr_at_3": 0.651075115360827, + "mrr_at_5": 0.6629596593882293, + "nauc_map_at_1000_diff1": 0.6285220174472382, + "nauc_map_at_1000_max": 0.41299190980173595, + "nauc_map_at_1000_std": -0.1412717477320673, + "nauc_map_at_100_diff1": 0.6284421575331365, + "nauc_map_at_100_max": 0.4130453087017874, + "nauc_map_at_100_std": -0.14117235254936086, + "nauc_map_at_10_diff1": 0.6279859060038745, + "nauc_map_at_10_max": 0.41400416478444313, + "nauc_map_at_10_std": -0.14186777535682646, + "nauc_map_at_1_diff1": 0.6602682400082788, + "nauc_map_at_1_max": 0.3912316390336524, + "nauc_map_at_1_std": -0.14337708632295212, + "nauc_map_at_20_diff1": 0.6280504624897899, + "nauc_map_at_20_max": 0.41336998980120987, + "nauc_map_at_20_std": -0.14154389012544227, + "nauc_map_at_3_diff1": 0.6277421136025821, + "nauc_map_at_3_max": 0.41222077246283667, + "nauc_map_at_3_std": -0.1489353979915137, + "nauc_map_at_5_diff1": 0.6280213679044305, + "nauc_map_at_5_max": 0.4142631361742392, + "nauc_map_at_5_std": -0.14461596915163746, + "nauc_mrr_at_1000_diff1": 0.6280604336582117, + "nauc_mrr_at_1000_max": 0.4127895133785666, + "nauc_mrr_at_1000_std": -0.14154612283140922, + "nauc_mrr_at_100_diff1": 0.6279809373212438, + "nauc_mrr_at_100_max": 0.4128431058706312, + "nauc_mrr_at_100_std": -0.14144646807203143, + "nauc_mrr_at_10_diff1": 0.627552470345681, + "nauc_mrr_at_10_max": 0.4138093128712975, + "nauc_mrr_at_10_std": -0.14213926062298227, + "nauc_mrr_at_1_diff1": 0.6597637048495609, + "nauc_mrr_at_1_max": 0.39106281976611684, + "nauc_mrr_at_1_std": -0.14387536833769987, + "nauc_mrr_at_20_diff1": 0.6275915307701843, + "nauc_mrr_at_20_max": 0.41316869101394144, + "nauc_mrr_at_20_std": -0.1418167268885182, + "nauc_mrr_at_3_diff1": 0.6272917175599597, + "nauc_mrr_at_3_max": 0.41194235184121264, + "nauc_mrr_at_3_std": -0.14931376270977983, + "nauc_mrr_at_5_diff1": 0.6275955065535904, + "nauc_mrr_at_5_max": 0.4140727424751866, + "nauc_mrr_at_5_std": -0.14489894947725457, + "nauc_ndcg_at_1000_diff1": 0.6216150449220736, + "nauc_ndcg_at_1000_max": 0.4186529417982135, + "nauc_ndcg_at_1000_std": -0.13307295081902534, + "nauc_ndcg_at_100_diff1": 0.6192397539889876, + "nauc_ndcg_at_100_max": 0.4201292566017396, + "nauc_ndcg_at_100_std": -0.12969030886277913, + "nauc_ndcg_at_10_diff1": 0.6161304313219622, + "nauc_ndcg_at_10_max": 0.4242917697287512, + "nauc_ndcg_at_10_std": -0.1345300355241636, + "nauc_ndcg_at_1_diff1": 0.6602682400082788, + "nauc_ndcg_at_1_max": 0.3912316390336524, + "nauc_ndcg_at_1_std": -0.14337708632295212, + "nauc_ndcg_at_20_diff1": 0.6159498672918121, + "nauc_ndcg_at_20_max": 0.42202037569360495, + "nauc_ndcg_at_20_std": -0.1329387468532144, + "nauc_ndcg_at_3_diff1": 0.617218270969091, + "nauc_ndcg_at_3_max": 0.42018988303764077, + "nauc_ndcg_at_3_std": -0.15042571773164942, + "nauc_ndcg_at_5_diff1": 0.6170224079479988, + "nauc_ndcg_at_5_max": 0.4245882524578618, + "nauc_ndcg_at_5_std": -0.14216194796615703, + "nauc_precision_at_1000_diff1": 0.5306101247673323, + "nauc_precision_at_1000_max": 0.5408593574160027, + "nauc_precision_at_1000_std": 0.23389581434963005, + "nauc_precision_at_100_diff1": 0.5226886622886234, + "nauc_precision_at_100_max": 0.4976558275048605, + "nauc_precision_at_100_std": 0.07773834330811694, + "nauc_precision_at_10_diff1": 0.5536321547664915, + "nauc_precision_at_10_max": 0.4790701361014338, + "nauc_precision_at_10_std": -0.08795519244813163, + "nauc_precision_at_1_diff1": 0.6602682400082788, + "nauc_precision_at_1_max": 0.3912316390336524, + "nauc_precision_at_1_std": -0.14337708632295212, + "nauc_precision_at_20_diff1": 0.5341293632067683, + "nauc_precision_at_20_max": 0.47662360767359546, + "nauc_precision_at_20_std": -0.06206054956377274, + "nauc_precision_at_3_diff1": 0.5801503209215977, + "nauc_precision_at_3_max": 0.44877981885233914, + "nauc_precision_at_3_std": -0.1555389368557497, + "nauc_precision_at_5_diff1": 0.5719604469660163, + "nauc_precision_at_5_max": 0.46837459005792337, + "nauc_precision_at_5_std": -0.1300416814508876, + "nauc_recall_at_1000_diff1": 0.5306101247673266, + "nauc_recall_at_1000_max": 0.5408593574159917, + "nauc_recall_at_1000_std": 0.23389581434963236, + "nauc_recall_at_100_diff1": 0.5226886622886217, + "nauc_recall_at_100_max": 0.497655827504859, + "nauc_recall_at_100_std": 0.0777383433081195, + "nauc_recall_at_10_diff1": 0.5536321547664931, + "nauc_recall_at_10_max": 0.4790701361014339, + "nauc_recall_at_10_std": -0.08795519244813191, + "nauc_recall_at_1_diff1": 0.6602682400082788, + "nauc_recall_at_1_max": 0.3912316390336524, + "nauc_recall_at_1_std": -0.14337708632295212, + "nauc_recall_at_20_diff1": 0.5341293632067676, + "nauc_recall_at_20_max": 0.47662360767359724, + "nauc_recall_at_20_std": -0.06206054956377155, + "nauc_recall_at_3_diff1": 0.5801503209215972, + "nauc_recall_at_3_max": 0.4487798188523378, + "nauc_recall_at_3_std": -0.1555389368557505, + "nauc_recall_at_5_diff1": 0.5719604469660166, + "nauc_recall_at_5_max": 0.46837459005792353, + "nauc_recall_at_5_std": -0.13004168145088707, + "ndcg_at_1": 0.5735, + "ndcg_at_10": 0.71298, + "ndcg_at_100": 0.73368, + "ndcg_at_1000": 0.73873, + "ndcg_at_20": 0.72251, + "ndcg_at_3": 0.67551, + "ndcg_at_5": 0.69675, + "precision_at_1": 0.5735, + "precision_at_10": 0.08474, + "precision_at_100": 0.00945, + "precision_at_1000": 0.00098, + "precision_at_20": 0.04424, + "precision_at_3": 0.24873, + "precision_at_5": 0.15948, + "recall_at_1": 0.5735, + "recall_at_10": 0.84737, + "recall_at_100": 0.94456, + "recall_at_1000": 0.98459, + "recall_at_20": 0.88476, + "recall_at_3": 0.74618, + "recall_at_5": 0.79742 + } + ] + }, + "task_name": "CodeSearchNetCCRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CodeSearchNetRetrieval.json b/results/voyageai__voyage-3/no_revision_available/CodeSearchNetRetrieval.json new file mode 100644 index 000000000..848783c9d --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CodeSearchNetRetrieval.json @@ -0,0 +1,893 @@ +{ + "dataset_revision": "fdc6a9e39575768c27eb8a2a5f702bf846eb4759", + "evaluation_time": 51.30580377578735, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "python", + "languages": [ + "python-Code" + ], + "main_score": 0.93045, + "map_at_1": 0.853, + "map_at_10": 0.9095, + "map_at_100": 0.90956, + "map_at_1000": 0.90957, + "map_at_20": 0.9095, + "map_at_3": 0.90383, + "map_at_5": 0.90833, + "mrr_at_1": 0.853, + "mrr_at_10": 0.9095027777777783, + "mrr_at_100": 0.9095565797863866, + "mrr_at_1000": 0.9095746402378327, + "mrr_at_20": 0.9095027777777783, + "mrr_at_3": 0.9038333333333338, + "mrr_at_5": 0.908333333333334, + "nauc_map_at_1000_diff1": 0.8576231292312056, + "nauc_map_at_1000_max": 0.476579913397626, + "nauc_map_at_1000_std": -0.04001447495210067, + "nauc_map_at_100_diff1": 0.8576349194381042, + "nauc_map_at_100_max": 0.4766771079275271, + "nauc_map_at_100_std": -0.03993585399029094, + "nauc_map_at_10_diff1": 0.8577225316618214, + "nauc_map_at_10_max": 0.47699916360379024, + "nauc_map_at_10_std": -0.039513705900334366, + "nauc_map_at_1_diff1": 0.8566821559490119, + "nauc_map_at_1_max": 0.4348290102547437, + "nauc_map_at_1_std": -0.06304646121823952, + "nauc_map_at_20_diff1": 0.8577225316618214, + "nauc_map_at_20_max": 0.47699916360379024, + "nauc_map_at_20_std": -0.039513705900334366, + "nauc_map_at_3_diff1": 0.8560830587720043, + "nauc_map_at_3_max": 0.4837678673814455, + "nauc_map_at_3_std": -0.043828680417299475, + "nauc_map_at_5_diff1": 0.8573294865714643, + "nauc_map_at_5_max": 0.47850380284906396, + "nauc_map_at_5_std": -0.03994142031581979, + "nauc_mrr_at_1000_diff1": 0.8576231292312056, + "nauc_mrr_at_1000_max": 0.476579913397626, + "nauc_mrr_at_1000_std": -0.04001447495210067, + "nauc_mrr_at_100_diff1": 0.8576349194381042, + "nauc_mrr_at_100_max": 0.4766771079275271, + "nauc_mrr_at_100_std": -0.03993585399029094, + "nauc_mrr_at_10_diff1": 0.8577225316618214, + "nauc_mrr_at_10_max": 0.47699916360379024, + "nauc_mrr_at_10_std": -0.039513705900334366, + "nauc_mrr_at_1_diff1": 0.8566821559490119, + "nauc_mrr_at_1_max": 0.4348290102547437, + "nauc_mrr_at_1_std": -0.06304646121823952, + "nauc_mrr_at_20_diff1": 0.8577225316618214, + "nauc_mrr_at_20_max": 0.47699916360379024, + "nauc_mrr_at_20_std": -0.039513705900334366, + "nauc_mrr_at_3_diff1": 0.8560830587720043, + "nauc_mrr_at_3_max": 0.4837678673814455, + "nauc_mrr_at_3_std": -0.043828680417299475, + "nauc_mrr_at_5_diff1": 0.8573294865714643, + "nauc_mrr_at_5_max": 0.47850380284906396, + "nauc_mrr_at_5_std": -0.03994142031581979, + "nauc_ndcg_at_1000_diff1": 0.8581673981896814, + "nauc_ndcg_at_1000_max": 0.48532369933139985, + "nauc_ndcg_at_1000_std": -0.030227107505034465, + "nauc_ndcg_at_100_diff1": 0.8584490704334455, + "nauc_ndcg_at_100_max": 0.4897275277298232, + "nauc_ndcg_at_100_std": -0.024388795014133102, + "nauc_ndcg_at_10_diff1": 0.8592493403024647, + "nauc_ndcg_at_10_max": 0.4926123952880877, + "nauc_ndcg_at_10_std": -0.020630981927553086, + "nauc_ndcg_at_1_diff1": 0.8566821559490119, + "nauc_ndcg_at_1_max": 0.4348290102547437, + "nauc_ndcg_at_1_std": -0.06304646121823952, + "nauc_ndcg_at_20_diff1": 0.8592493403024647, + "nauc_ndcg_at_20_max": 0.4926123952880877, + "nauc_ndcg_at_20_std": -0.020630981927553086, + "nauc_ndcg_at_3_diff1": 0.8556809834455305, + "nauc_ndcg_at_3_max": 0.5080682175013136, + "nauc_ndcg_at_3_std": -0.02973172992603311, + "nauc_ndcg_at_5_diff1": 0.858326545946113, + "nauc_ndcg_at_5_max": 0.49847938118727747, + "nauc_ndcg_at_5_std": -0.02053388760469525, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8846872082166319, + "nauc_precision_at_100_max": 0.9738562091503591, + "nauc_precision_at_100_std": 0.661157796451902, + "nauc_precision_at_10_diff1": 0.9176337201547365, + "nauc_precision_at_10_max": 0.9813258636787986, + "nauc_precision_at_10_std": 0.6545951714018879, + "nauc_precision_at_1_diff1": 0.8566821559490119, + "nauc_precision_at_1_max": 0.4348290102547437, + "nauc_precision_at_1_std": -0.06304646121823952, + "nauc_precision_at_20_diff1": 0.9176337201547365, + "nauc_precision_at_20_max": 0.9813258636787986, + "nauc_precision_at_20_std": 0.6545951714018879, + "nauc_precision_at_3_diff1": 0.8525543550753646, + "nauc_precision_at_3_max": 0.6919301053754875, + "nauc_precision_at_3_std": 0.08060557556356368, + "nauc_precision_at_5_diff1": 0.8755543884220438, + "nauc_precision_at_5_max": 0.8039507469654557, + "nauc_precision_at_5_std": 0.2966853408029781, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8846872082166171, + "nauc_recall_at_100_max": 0.9738562091503128, + "nauc_recall_at_100_std": 0.6611577964519142, + "nauc_recall_at_10_diff1": 0.917633720154742, + "nauc_recall_at_10_max": 0.9813258636788128, + "nauc_recall_at_10_std": 0.6545951714018913, + "nauc_recall_at_1_diff1": 0.8566821559490119, + "nauc_recall_at_1_max": 0.4348290102547437, + "nauc_recall_at_1_std": -0.06304646121823952, + "nauc_recall_at_20_diff1": 0.917633720154742, + "nauc_recall_at_20_max": 0.9813258636788128, + "nauc_recall_at_20_std": 0.6545951714018913, + "nauc_recall_at_3_diff1": 0.8525543550753665, + "nauc_recall_at_3_max": 0.6919301053754875, + "nauc_recall_at_3_std": 0.08060557556355995, + "nauc_recall_at_5_diff1": 0.8755543884220373, + "nauc_recall_at_5_max": 0.8039507469654529, + "nauc_recall_at_5_std": 0.2966853408029875, + "ndcg_at_1": 0.853, + "ndcg_at_10": 0.93045, + "ndcg_at_100": 0.93083, + "ndcg_at_1000": 0.93143, + "ndcg_at_20": 0.93045, + "ndcg_at_3": 0.91961, + "ndcg_at_5": 0.92757, + "precision_at_1": 0.853, + "precision_at_10": 0.0993, + "precision_at_100": 0.00995, + "precision_at_1000": 0.001, + "precision_at_20": 0.04965, + "precision_at_3": 0.32167, + "precision_at_5": 0.1968, + "recall_at_1": 0.853, + "recall_at_10": 0.993, + "recall_at_100": 0.995, + "recall_at_1000": 1.0, + "recall_at_20": 0.993, + "recall_at_3": 0.965, + "recall_at_5": 0.984 + }, + { + "hf_subset": "javascript", + "languages": [ + "javascript-Code" + ], + "main_score": 0.87256, + "map_at_1": 0.801, + "map_at_10": 0.85029, + "map_at_100": 0.85234, + "map_at_1000": 0.85239, + "map_at_20": 0.85178, + "map_at_3": 0.84033, + "map_at_5": 0.84623, + "mrr_at_1": 0.801, + "mrr_at_10": 0.8502876984126986, + "mrr_at_100": 0.8523392886319414, + "mrr_at_1000": 0.8523875492324384, + "mrr_at_20": 0.8517779617011819, + "mrr_at_3": 0.8403333333333336, + "mrr_at_5": 0.8462333333333337, + "nauc_map_at_1000_diff1": 0.8842570743097193, + "nauc_map_at_1000_max": 0.7587240204547016, + "nauc_map_at_1000_std": 0.08228179893758535, + "nauc_map_at_100_diff1": 0.8842582493663028, + "nauc_map_at_100_max": 0.7588070039632743, + "nauc_map_at_100_std": 0.08256009136316146, + "nauc_map_at_10_diff1": 0.8841453035578714, + "nauc_map_at_10_max": 0.7607353545779308, + "nauc_map_at_10_std": 0.08359279021631655, + "nauc_map_at_1_diff1": 0.8870973407805525, + "nauc_map_at_1_max": 0.7124816203222778, + "nauc_map_at_1_std": 0.04174758743206364, + "nauc_map_at_20_diff1": 0.8841228294774051, + "nauc_map_at_20_max": 0.7596054398983038, + "nauc_map_at_20_std": 0.08443849693729685, + "nauc_map_at_3_diff1": 0.884992079090233, + "nauc_map_at_3_max": 0.7607116727362903, + "nauc_map_at_3_std": 0.06816439832851451, + "nauc_map_at_5_diff1": 0.882872655258455, + "nauc_map_at_5_max": 0.7581830680162894, + "nauc_map_at_5_std": 0.07563533783684627, + "nauc_mrr_at_1000_diff1": 0.8842569162850039, + "nauc_mrr_at_1000_max": 0.7587241282755939, + "nauc_mrr_at_1000_std": 0.08228242304037472, + "nauc_mrr_at_100_diff1": 0.8842580913947929, + "nauc_mrr_at_100_max": 0.7588071116661819, + "nauc_mrr_at_100_std": 0.082560714979987, + "nauc_mrr_at_10_diff1": 0.8841453035578714, + "nauc_mrr_at_10_max": 0.7607353545779308, + "nauc_mrr_at_10_std": 0.08359279021631655, + "nauc_mrr_at_1_diff1": 0.8870973407805525, + "nauc_mrr_at_1_max": 0.7124816203222778, + "nauc_mrr_at_1_std": 0.04174758743206364, + "nauc_mrr_at_20_diff1": 0.8841228294774051, + "nauc_mrr_at_20_max": 0.7596054398983038, + "nauc_mrr_at_20_std": 0.08443849693729685, + "nauc_mrr_at_3_diff1": 0.884992079090233, + "nauc_mrr_at_3_max": 0.7607116727362903, + "nauc_mrr_at_3_std": 0.06816439832851451, + "nauc_mrr_at_5_diff1": 0.882872655258455, + "nauc_mrr_at_5_max": 0.7581830680162894, + "nauc_mrr_at_5_std": 0.07563533783684627, + "nauc_ndcg_at_1000_diff1": 0.8834007378444099, + "nauc_ndcg_at_1000_max": 0.7679191393287695, + "nauc_ndcg_at_1000_std": 0.09857767842891535, + "nauc_ndcg_at_100_diff1": 0.8834809252528804, + "nauc_ndcg_at_100_max": 0.7713766804531194, + "nauc_ndcg_at_100_std": 0.11044342428924596, + "nauc_ndcg_at_10_diff1": 0.8821667688437236, + "nauc_ndcg_at_10_max": 0.7817373657424341, + "nauc_ndcg_at_10_std": 0.11658965417210362, + "nauc_ndcg_at_1_diff1": 0.8870973407805525, + "nauc_ndcg_at_1_max": 0.7124816203222778, + "nauc_ndcg_at_1_std": 0.04174758743206364, + "nauc_ndcg_at_20_diff1": 0.8818803925975053, + "nauc_ndcg_at_20_max": 0.7780537878106746, + "nauc_ndcg_at_20_std": 0.12374343532243792, + "nauc_ndcg_at_3_diff1": 0.8834462766104135, + "nauc_ndcg_at_3_max": 0.778422523841922, + "nauc_ndcg_at_3_std": 0.0783961450812364, + "nauc_ndcg_at_5_diff1": 0.8792638440451687, + "nauc_ndcg_at_5_max": 0.774577567769517, + "nauc_ndcg_at_5_std": 0.09365186658740293, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8911231159130302, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.8976590636254534, + "nauc_precision_at_10_diff1": 0.8683634373289549, + "nauc_precision_at_10_max": 0.9309378923983399, + "nauc_precision_at_10_std": 0.37337808686692875, + "nauc_precision_at_1_diff1": 0.8870973407805525, + "nauc_precision_at_1_max": 0.7124816203222778, + "nauc_precision_at_1_std": 0.04174758743206364, + "nauc_precision_at_20_diff1": 0.8563605622429126, + "nauc_precision_at_20_max": 0.9717364423246777, + "nauc_precision_at_20_std": 0.6202967673555915, + "nauc_precision_at_3_diff1": 0.87714552083507, + "nauc_precision_at_3_max": 0.8466295969500727, + "nauc_precision_at_3_std": 0.11803902309088168, + "nauc_precision_at_5_diff1": 0.8608299380358202, + "nauc_precision_at_5_max": 0.8510525422290129, + "nauc_precision_at_5_std": 0.18274127832951048, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8911231159130345, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.8976590636254583, + "nauc_recall_at_10_diff1": 0.868363437328955, + "nauc_recall_at_10_max": 0.9309378923983386, + "nauc_recall_at_10_std": 0.37337808686693136, + "nauc_recall_at_1_diff1": 0.8870973407805525, + "nauc_recall_at_1_max": 0.7124816203222778, + "nauc_recall_at_1_std": 0.04174758743206364, + "nauc_recall_at_20_diff1": 0.8563605622429136, + "nauc_recall_at_20_max": 0.9717364423246795, + "nauc_recall_at_20_std": 0.6202967673555918, + "nauc_recall_at_3_diff1": 0.8771455208350678, + "nauc_recall_at_3_max": 0.8466295969500728, + "nauc_recall_at_3_std": 0.11803902309088278, + "nauc_recall_at_5_diff1": 0.8608299380358204, + "nauc_recall_at_5_max": 0.8510525422290128, + "nauc_recall_at_5_std": 0.18274127832951242, + "ndcg_at_1": 0.801, + "ndcg_at_10": 0.87256, + "ndcg_at_100": 0.88214, + "ndcg_at_1000": 0.88381, + "ndcg_at_20": 0.87792, + "ndcg_at_3": 0.85214, + "ndcg_at_5": 0.86281, + "precision_at_1": 0.801, + "precision_at_10": 0.0942, + "precision_at_100": 0.00986, + "precision_at_1000": 0.001, + "precision_at_20": 0.04815, + "precision_at_3": 0.29533, + "precision_at_5": 0.1824, + "recall_at_1": 0.801, + "recall_at_10": 0.942, + "recall_at_100": 0.986, + "recall_at_1000": 1.0, + "recall_at_20": 0.963, + "recall_at_3": 0.886, + "recall_at_5": 0.912 + }, + { + "hf_subset": "go", + "languages": [ + "go-Code" + ], + "main_score": 0.97357, + "map_at_1": 0.938, + "map_at_10": 0.96518, + "map_at_100": 0.9652, + "map_at_1000": 0.96521, + "map_at_20": 0.96518, + "map_at_3": 0.96433, + "map_at_5": 0.96478, + "mrr_at_1": 0.938, + "mrr_at_10": 0.9651761904761905, + "mrr_at_100": 0.965204761904762, + "mrr_at_1000": 0.9652072935503316, + "mrr_at_20": 0.9651761904761905, + "mrr_at_3": 0.9643333333333334, + "mrr_at_5": 0.9647833333333334, + "nauc_map_at_1000_diff1": 0.9571564449710245, + "nauc_map_at_1000_max": 0.5809928586846871, + "nauc_map_at_1000_std": -0.016576214572068346, + "nauc_map_at_100_diff1": 0.9571500512982878, + "nauc_map_at_100_max": 0.5810233449704861, + "nauc_map_at_100_std": -0.016616007289732494, + "nauc_map_at_10_diff1": 0.9570779585821289, + "nauc_map_at_10_max": 0.5813670971830122, + "nauc_map_at_10_std": -0.01802993862661893, + "nauc_map_at_1_diff1": 0.956703111352068, + "nauc_map_at_1_max": 0.5398783169181656, + "nauc_map_at_1_std": 0.000587331706878942, + "nauc_map_at_20_diff1": 0.9570779585821289, + "nauc_map_at_20_max": 0.5813670971830122, + "nauc_map_at_20_std": -0.01802993862661893, + "nauc_map_at_3_diff1": 0.9580922711763828, + "nauc_map_at_3_max": 0.583049730795748, + "nauc_map_at_3_std": -0.014948035288882533, + "nauc_map_at_5_diff1": 0.9575567725118124, + "nauc_map_at_5_max": 0.5860371282130169, + "nauc_map_at_5_std": -0.011064624619369374, + "nauc_mrr_at_1000_diff1": 0.9571564449710245, + "nauc_mrr_at_1000_max": 0.5809928586846871, + "nauc_mrr_at_1000_std": -0.016576214572068346, + "nauc_mrr_at_100_diff1": 0.9571500512982878, + "nauc_mrr_at_100_max": 0.5810233449704861, + "nauc_mrr_at_100_std": -0.016616007289732494, + "nauc_mrr_at_10_diff1": 0.9570779585821289, + "nauc_mrr_at_10_max": 0.5813670971830122, + "nauc_mrr_at_10_std": -0.01802993862661893, + "nauc_mrr_at_1_diff1": 0.956703111352068, + "nauc_mrr_at_1_max": 0.5398783169181656, + "nauc_mrr_at_1_std": 0.000587331706878942, + "nauc_mrr_at_20_diff1": 0.9570779585821289, + "nauc_mrr_at_20_max": 0.5813670971830122, + "nauc_mrr_at_20_std": -0.01802993862661893, + "nauc_mrr_at_3_diff1": 0.9580922711763828, + "nauc_mrr_at_3_max": 0.583049730795748, + "nauc_mrr_at_3_std": -0.014948035288882533, + "nauc_mrr_at_5_diff1": 0.9575567725118124, + "nauc_mrr_at_5_max": 0.5860371282130169, + "nauc_mrr_at_5_std": -0.011064624619369374, + "nauc_ndcg_at_1000_diff1": 0.9566546318673345, + "nauc_ndcg_at_1000_max": 0.5873436212244307, + "nauc_ndcg_at_1000_std": -0.023132590579786776, + "nauc_ndcg_at_100_diff1": 0.95626876687286, + "nauc_ndcg_at_100_max": 0.5891660215446458, + "nauc_ndcg_at_100_std": -0.025518961728067235, + "nauc_ndcg_at_10_diff1": 0.955632238545921, + "nauc_ndcg_at_10_max": 0.592172278767897, + "nauc_ndcg_at_10_std": -0.03806431823992229, + "nauc_ndcg_at_1_diff1": 0.956703111352068, + "nauc_ndcg_at_1_max": 0.5398783169181656, + "nauc_ndcg_at_1_std": 0.000587331706878942, + "nauc_ndcg_at_20_diff1": 0.955632238545921, + "nauc_ndcg_at_20_max": 0.592172278767897, + "nauc_ndcg_at_20_std": -0.03806431823992229, + "nauc_ndcg_at_3_diff1": 0.9584340496636652, + "nauc_ndcg_at_3_max": 0.6000473386549824, + "nauc_ndcg_at_3_std": -0.022727589676426126, + "nauc_ndcg_at_5_diff1": 0.957193755467418, + "nauc_ndcg_at_5_max": 0.6065257161937733, + "nauc_ndcg_at_5_std": -0.015498998286315199, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8692810457515913, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": -0.5634920634918772, + "nauc_precision_at_10_diff1": 0.8692810457515859, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": -1.1517273576098859, + "nauc_precision_at_1_diff1": 0.956703111352068, + "nauc_precision_at_1_max": 0.5398783169181656, + "nauc_precision_at_1_std": 0.000587331706878942, + "nauc_precision_at_20_diff1": 0.8692810457515859, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": -1.1517273576098859, + "nauc_precision_at_3_diff1": 0.962651727357616, + "nauc_precision_at_3_max": 0.8326663998932856, + "nauc_precision_at_3_std": -0.1329198346004975, + "nauc_precision_at_5_diff1": 0.9477124183006679, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": -0.10018674136322904, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8692810457516946, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": -0.563492063491993, + "nauc_recall_at_10_diff1": 0.8692810457516293, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": -1.1517273576096938, + "nauc_recall_at_1_diff1": 0.956703111352068, + "nauc_recall_at_1_max": 0.5398783169181656, + "nauc_recall_at_1_std": 0.000587331706878942, + "nauc_recall_at_20_diff1": 0.8692810457516293, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": -1.1517273576096938, + "nauc_recall_at_3_diff1": 0.9626517273576256, + "nauc_recall_at_3_max": 0.8326663998933058, + "nauc_recall_at_3_std": -0.13291983460052423, + "nauc_recall_at_5_diff1": 0.9477124183006517, + "nauc_recall_at_5_max": 1.0, + "nauc_recall_at_5_std": -0.10018674136320944, + "ndcg_at_1": 0.938, + "ndcg_at_10": 0.97357, + "ndcg_at_100": 0.97376, + "ndcg_at_1000": 0.97388, + "ndcg_at_20": 0.97357, + "ndcg_at_3": 0.97178, + "ndcg_at_5": 0.9726, + "precision_at_1": 0.938, + "precision_at_10": 0.0998, + "precision_at_100": 0.00999, + "precision_at_1000": 0.001, + "precision_at_20": 0.0499, + "precision_at_3": 0.331, + "precision_at_5": 0.199, + "recall_at_1": 0.938, + "recall_at_10": 0.998, + "recall_at_100": 0.999, + "recall_at_1000": 1.0, + "recall_at_20": 0.998, + "recall_at_3": 0.993, + "recall_at_5": 0.995 + }, + { + "hf_subset": "ruby", + "languages": [ + "ruby-Code" + ], + "main_score": 0.90889, + "map_at_1": 0.836, + "map_at_10": 0.88911, + "map_at_100": 0.89002, + "map_at_1000": 0.89008, + "map_at_20": 0.88974, + "map_at_3": 0.88383, + "map_at_5": 0.88728, + "mrr_at_1": 0.836, + "mrr_at_10": 0.8891107142857149, + "mrr_at_100": 0.8900222994146085, + "mrr_at_1000": 0.890080336750091, + "mrr_at_20": 0.889736920329413, + "mrr_at_3": 0.8838333333333337, + "mrr_at_5": 0.8872833333333339, + "nauc_map_at_1000_diff1": 0.8453209449081388, + "nauc_map_at_1000_max": 0.7535488021194996, + "nauc_map_at_1000_std": 0.24677044872722692, + "nauc_map_at_100_diff1": 0.8453713697475752, + "nauc_map_at_100_max": 0.7536840014409799, + "nauc_map_at_100_std": 0.24718054650826637, + "nauc_map_at_10_diff1": 0.8447342916754987, + "nauc_map_at_10_max": 0.7554013455616437, + "nauc_map_at_10_std": 0.2523112462080545, + "nauc_map_at_1_diff1": 0.854122171294185, + "nauc_map_at_1_max": 0.7132928175469749, + "nauc_map_at_1_std": 0.14337308919872335, + "nauc_map_at_20_diff1": 0.8446963820905934, + "nauc_map_at_20_max": 0.7542468915483412, + "nauc_map_at_20_std": 0.24869792018464928, + "nauc_map_at_3_diff1": 0.8404309574531664, + "nauc_map_at_3_max": 0.7569388569038381, + "nauc_map_at_3_std": 0.24255531144082795, + "nauc_map_at_5_diff1": 0.8440059056204763, + "nauc_map_at_5_max": 0.7572229790405535, + "nauc_map_at_5_std": 0.25076662805387673, + "nauc_mrr_at_1000_diff1": 0.8453209449081388, + "nauc_mrr_at_1000_max": 0.7535488021194996, + "nauc_mrr_at_1000_std": 0.24677044872722692, + "nauc_mrr_at_100_diff1": 0.8453713697475752, + "nauc_mrr_at_100_max": 0.7536840014409799, + "nauc_mrr_at_100_std": 0.24718054650826637, + "nauc_mrr_at_10_diff1": 0.8447342916754987, + "nauc_mrr_at_10_max": 0.7554013455616437, + "nauc_mrr_at_10_std": 0.2523112462080545, + "nauc_mrr_at_1_diff1": 0.854122171294185, + "nauc_mrr_at_1_max": 0.7132928175469749, + "nauc_mrr_at_1_std": 0.14337308919872335, + "nauc_mrr_at_20_diff1": 0.8446963820905934, + "nauc_mrr_at_20_max": 0.7542468915483412, + "nauc_mrr_at_20_std": 0.24869792018464928, + "nauc_mrr_at_3_diff1": 0.8404309574531664, + "nauc_mrr_at_3_max": 0.7569388569038381, + "nauc_mrr_at_3_std": 0.24255531144082795, + "nauc_mrr_at_5_diff1": 0.8440059056204763, + "nauc_mrr_at_5_max": 0.7572229790405535, + "nauc_mrr_at_5_std": 0.25076662805387673, + "nauc_ndcg_at_1000_diff1": 0.8458913383195665, + "nauc_ndcg_at_1000_max": 0.7628703800449987, + "nauc_ndcg_at_1000_std": 0.27228953623611923, + "nauc_ndcg_at_100_diff1": 0.8476583367436076, + "nauc_ndcg_at_100_max": 0.767193679672607, + "nauc_ndcg_at_100_std": 0.2853761871982123, + "nauc_ndcg_at_10_diff1": 0.8444541017327882, + "nauc_ndcg_at_10_max": 0.7767844584441539, + "nauc_ndcg_at_10_std": 0.3134738961755448, + "nauc_ndcg_at_1_diff1": 0.854122171294185, + "nauc_ndcg_at_1_max": 0.7132928175469749, + "nauc_ndcg_at_1_std": 0.14337308919872335, + "nauc_ndcg_at_20_diff1": 0.8439641232380867, + "nauc_ndcg_at_20_max": 0.7720990183835106, + "nauc_ndcg_at_20_std": 0.29872075565706496, + "nauc_ndcg_at_3_diff1": 0.8352350067449279, + "nauc_ndcg_at_3_max": 0.7787170129692527, + "nauc_ndcg_at_3_std": 0.2867526739578895, + "nauc_ndcg_at_5_diff1": 0.8425558969892223, + "nauc_ndcg_at_5_max": 0.7807758722432178, + "nauc_ndcg_at_5_std": 0.3071561752705588, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.9441721132897686, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.9891067538126498, + "nauc_precision_at_10_diff1": 0.8485206582633057, + "nauc_precision_at_10_max": 0.9790645424836606, + "nauc_precision_at_10_std": 0.9098535247432278, + "nauc_precision_at_1_diff1": 0.854122171294185, + "nauc_precision_at_1_max": 0.7132928175469749, + "nauc_precision_at_1_std": 0.14337308919872335, + "nauc_precision_at_20_diff1": 0.8406000081191887, + "nauc_precision_at_20_max": 0.988633134413182, + "nauc_precision_at_20_std": 0.9247148134616135, + "nauc_precision_at_3_diff1": 0.8076643086613199, + "nauc_precision_at_3_max": 0.8978857079554998, + "nauc_precision_at_3_std": 0.5258114545253111, + "nauc_precision_at_5_diff1": 0.8358798064680434, + "nauc_precision_at_5_max": 0.9489113827349118, + "nauc_precision_at_5_std": 0.7109859095153187, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9441721132897544, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.9891067538126302, + "nauc_recall_at_10_diff1": 0.8485206582633095, + "nauc_recall_at_10_max": 0.9790645424836617, + "nauc_recall_at_10_std": 0.9098535247432313, + "nauc_recall_at_1_diff1": 0.854122171294185, + "nauc_recall_at_1_max": 0.7132928175469749, + "nauc_recall_at_1_std": 0.14337308919872335, + "nauc_recall_at_20_diff1": 0.8406000081191933, + "nauc_recall_at_20_max": 0.988633134413185, + "nauc_recall_at_20_std": 0.9247148134616201, + "nauc_recall_at_3_diff1": 0.8076643086613168, + "nauc_recall_at_3_max": 0.8978857079554988, + "nauc_recall_at_3_std": 0.5258114545253126, + "nauc_recall_at_5_diff1": 0.8358798064680442, + "nauc_recall_at_5_max": 0.9489113827349114, + "nauc_recall_at_5_std": 0.710985909515323, + "ndcg_at_1": 0.836, + "ndcg_at_10": 0.90889, + "ndcg_at_100": 0.91321, + "ndcg_at_1000": 0.91473, + "ndcg_at_20": 0.91116, + "ndcg_at_3": 0.89858, + "ndcg_at_5": 0.90478, + "precision_at_1": 0.836, + "precision_at_10": 0.0968, + "precision_at_100": 0.00988, + "precision_at_1000": 0.001, + "precision_at_20": 0.04885, + "precision_at_3": 0.31367, + "precision_at_5": 0.1912, + "recall_at_1": 0.836, + "recall_at_10": 0.968, + "recall_at_100": 0.988, + "recall_at_1000": 1.0, + "recall_at_20": 0.977, + "recall_at_3": 0.941, + "recall_at_5": 0.956 + }, + { + "hf_subset": "java", + "languages": [ + "java-Code" + ], + "main_score": 0.93564, + "map_at_1": 0.878, + "map_at_10": 0.92074, + "map_at_100": 0.9213, + "map_at_1000": 0.92132, + "map_at_20": 0.92103, + "map_at_3": 0.917, + "map_at_5": 0.91945, + "mrr_at_1": 0.878, + "mrr_at_10": 0.9207396825396825, + "mrr_at_100": 0.9213006675278653, + "mrr_at_1000": 0.9213230436977999, + "mrr_at_20": 0.9210253800722843, + "mrr_at_3": 0.9170000000000001, + "mrr_at_5": 0.9194500000000002, + "nauc_map_at_1000_diff1": 0.8939853982839457, + "nauc_map_at_1000_max": 0.6363013257354126, + "nauc_map_at_1000_std": 0.26393209559157593, + "nauc_map_at_100_diff1": 0.8940161018436459, + "nauc_map_at_100_max": 0.6364066588100777, + "nauc_map_at_100_std": 0.26401614888220204, + "nauc_map_at_10_diff1": 0.8946512885363085, + "nauc_map_at_10_max": 0.6390276314698443, + "nauc_map_at_10_std": 0.26726599071925017, + "nauc_map_at_1_diff1": 0.8707184399053156, + "nauc_map_at_1_max": 0.5404955244470222, + "nauc_map_at_1_std": 0.1707215751438296, + "nauc_map_at_20_diff1": 0.8943622832862244, + "nauc_map_at_20_max": 0.6376975658386984, + "nauc_map_at_20_std": 0.26539147985594275, + "nauc_map_at_3_diff1": 0.897810225941485, + "nauc_map_at_3_max": 0.651968640649625, + "nauc_map_at_3_std": 0.28289049412536943, + "nauc_map_at_5_diff1": 0.8961326329832154, + "nauc_map_at_5_max": 0.6442860113109565, + "nauc_map_at_5_std": 0.2724320446568598, + "nauc_mrr_at_1000_diff1": 0.8939853982839457, + "nauc_mrr_at_1000_max": 0.6363013257354126, + "nauc_mrr_at_1000_std": 0.26393209559157593, + "nauc_mrr_at_100_diff1": 0.8940161018436459, + "nauc_mrr_at_100_max": 0.6364066588100777, + "nauc_mrr_at_100_std": 0.26401614888220204, + "nauc_mrr_at_10_diff1": 0.8946512885363085, + "nauc_mrr_at_10_max": 0.6390276314698443, + "nauc_mrr_at_10_std": 0.26726599071925017, + "nauc_mrr_at_1_diff1": 0.8707184399053156, + "nauc_mrr_at_1_max": 0.5404955244470222, + "nauc_mrr_at_1_std": 0.1707215751438296, + "nauc_mrr_at_20_diff1": 0.8943622832862244, + "nauc_mrr_at_20_max": 0.6376975658386984, + "nauc_mrr_at_20_std": 0.26539147985594275, + "nauc_mrr_at_3_diff1": 0.897810225941485, + "nauc_mrr_at_3_max": 0.651968640649625, + "nauc_mrr_at_3_std": 0.28289049412536943, + "nauc_mrr_at_5_diff1": 0.8961326329832154, + "nauc_mrr_at_5_max": 0.6442860113109565, + "nauc_mrr_at_5_std": 0.2724320446568598, + "nauc_ndcg_at_1000_diff1": 0.8977624187126672, + "nauc_ndcg_at_1000_max": 0.6509489741027457, + "nauc_ndcg_at_1000_std": 0.2776610724580472, + "nauc_ndcg_at_100_diff1": 0.8984817149282766, + "nauc_ndcg_at_100_max": 0.6534047352702234, + "nauc_ndcg_at_100_std": 0.2796063554924979, + "nauc_ndcg_at_10_diff1": 0.9029524045535421, + "nauc_ndcg_at_10_max": 0.6715677636871343, + "nauc_ndcg_at_10_std": 0.3000812119244512, + "nauc_ndcg_at_1_diff1": 0.8707184399053156, + "nauc_ndcg_at_1_max": 0.5404955244470222, + "nauc_ndcg_at_1_std": 0.1707215751438296, + "nauc_ndcg_at_20_diff1": 0.9018657206627876, + "nauc_ndcg_at_20_max": 0.6661866897520713, + "nauc_ndcg_at_20_std": 0.2922670148123546, + "nauc_ndcg_at_3_diff1": 0.9095669173706723, + "nauc_ndcg_at_3_max": 0.6978750306887669, + "nauc_ndcg_at_3_std": 0.33034781211110215, + "nauc_ndcg_at_5_diff1": 0.9066873940742384, + "nauc_ndcg_at_5_max": 0.6844076920091323, + "nauc_ndcg_at_5_std": 0.3112348484346909, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.5541549953314321, + "nauc_precision_at_10_diff1": 0.9869281045751674, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.6375116713352018, + "nauc_precision_at_1_diff1": 0.8707184399053156, + "nauc_precision_at_1_max": 0.5404955244470222, + "nauc_precision_at_1_std": 0.1707215751438296, + "nauc_precision_at_20_diff1": 0.9918300653594746, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.6026202147525709, + "nauc_precision_at_3_diff1": 0.9771241830065329, + "nauc_precision_at_3_max": 0.9599789915966325, + "nauc_precision_at_3_std": 0.6022175536881422, + "nauc_precision_at_5_diff1": 0.9864773495605154, + "nauc_precision_at_5_max": 0.9846260343217785, + "nauc_precision_at_5_std": 0.6003895811198082, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 1.0, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.554154995331434, + "nauc_recall_at_10_diff1": 0.9869281045751627, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.6375116713351952, + "nauc_recall_at_1_diff1": 0.8707184399053156, + "nauc_recall_at_1_max": 0.5404955244470222, + "nauc_recall_at_1_std": 0.1707215751438296, + "nauc_recall_at_20_diff1": 0.9918300653594807, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.6026202147525692, + "nauc_recall_at_3_diff1": 0.9771241830065364, + "nauc_recall_at_3_max": 0.9599789915966359, + "nauc_recall_at_3_std": 0.6022175536881402, + "nauc_recall_at_5_diff1": 0.9864773495605109, + "nauc_recall_at_5_max": 0.9846260343217718, + "nauc_recall_at_5_std": 0.6003895811198021, + "ndcg_at_1": 0.878, + "ndcg_at_10": 0.93564, + "ndcg_at_100": 0.93896, + "ndcg_at_1000": 0.93938, + "ndcg_at_20": 0.93666, + "ndcg_at_3": 0.92817, + "ndcg_at_5": 0.93264, + "precision_at_1": 0.878, + "precision_at_10": 0.098, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.0492, + "precision_at_3": 0.32, + "precision_at_5": 0.1942, + "recall_at_1": 0.878, + "recall_at_10": 0.98, + "recall_at_100": 0.997, + "recall_at_1000": 1.0, + "recall_at_20": 0.984, + "recall_at_3": 0.96, + "recall_at_5": 0.971 + }, + { + "hf_subset": "php", + "languages": [ + "php-Code" + ], + "main_score": 0.91689, + "map_at_1": 0.84, + "map_at_10": 0.8945, + "map_at_100": 0.89495, + "map_at_1000": 0.89498, + "map_at_20": 0.89481, + "map_at_3": 0.88633, + "map_at_5": 0.89248, + "mrr_at_1": 0.84, + "mrr_at_10": 0.8944996031746038, + "mrr_at_100": 0.8949508735851238, + "mrr_at_1000": 0.8949808536662898, + "mrr_at_20": 0.8948055305805312, + "mrr_at_3": 0.8863333333333339, + "mrr_at_5": 0.8924833333333341, + "nauc_map_at_1000_diff1": 0.9352047425803509, + "nauc_map_at_1000_max": 0.5475716762690978, + "nauc_map_at_1000_std": -0.0991522281714176, + "nauc_map_at_100_diff1": 0.9351874431925973, + "nauc_map_at_100_max": 0.5477058022992707, + "nauc_map_at_100_std": -0.09883857009416624, + "nauc_map_at_10_diff1": 0.9351254354268157, + "nauc_map_at_10_max": 0.5497151480644573, + "nauc_map_at_10_std": -0.0942704207160786, + "nauc_map_at_1_diff1": 0.9384664948453604, + "nauc_map_at_1_max": 0.4803448699067248, + "nauc_map_at_1_std": -0.154452012763868, + "nauc_map_at_20_diff1": 0.9352321425032711, + "nauc_map_at_20_max": 0.5483549176809716, + "nauc_map_at_20_std": -0.09726155888889265, + "nauc_map_at_3_diff1": 0.9344816242261298, + "nauc_map_at_3_max": 0.5643499493776413, + "nauc_map_at_3_std": -0.09845481281071826, + "nauc_map_at_5_diff1": 0.9351267631572749, + "nauc_map_at_5_max": 0.5548067221058777, + "nauc_map_at_5_std": -0.10044613048389034, + "nauc_mrr_at_1000_diff1": 0.9352047425803509, + "nauc_mrr_at_1000_max": 0.5475716762690978, + "nauc_mrr_at_1000_std": -0.0991522281714176, + "nauc_mrr_at_100_diff1": 0.9351874431925973, + "nauc_mrr_at_100_max": 0.5477058022992707, + "nauc_mrr_at_100_std": -0.09883857009416624, + "nauc_mrr_at_10_diff1": 0.9351254354268157, + "nauc_mrr_at_10_max": 0.5497151480644573, + "nauc_mrr_at_10_std": -0.0942704207160786, + "nauc_mrr_at_1_diff1": 0.9384664948453604, + "nauc_mrr_at_1_max": 0.4803448699067248, + "nauc_mrr_at_1_std": -0.154452012763868, + "nauc_mrr_at_20_diff1": 0.9352321425032711, + "nauc_mrr_at_20_max": 0.5483549176809716, + "nauc_mrr_at_20_std": -0.09726155888889265, + "nauc_mrr_at_3_diff1": 0.9344816242261298, + "nauc_mrr_at_3_max": 0.5643499493776413, + "nauc_mrr_at_3_std": -0.09845481281071826, + "nauc_mrr_at_5_diff1": 0.9351267631572749, + "nauc_mrr_at_5_max": 0.5548067221058777, + "nauc_mrr_at_5_std": -0.10044613048389034, + "nauc_ndcg_at_1000_diff1": 0.9349617039199085, + "nauc_ndcg_at_1000_max": 0.5585003967656083, + "nauc_ndcg_at_1000_std": -0.0821951339896603, + "nauc_ndcg_at_100_diff1": 0.93466077953014, + "nauc_ndcg_at_100_max": 0.5616866178408509, + "nauc_ndcg_at_100_std": -0.07463393121878209, + "nauc_ndcg_at_10_diff1": 0.9346622682163949, + "nauc_ndcg_at_10_max": 0.5741307040457725, + "nauc_ndcg_at_10_std": -0.04554685682220592, + "nauc_ndcg_at_1_diff1": 0.9384664948453604, + "nauc_ndcg_at_1_max": 0.4803448699067248, + "nauc_ndcg_at_1_std": -0.154452012763868, + "nauc_ndcg_at_20_diff1": 0.9351047809543326, + "nauc_ndcg_at_20_max": 0.5684953979731645, + "nauc_ndcg_at_20_std": -0.057940518564235925, + "nauc_ndcg_at_3_diff1": 0.9333835295391505, + "nauc_ndcg_at_3_max": 0.6033528695875281, + "nauc_ndcg_at_3_std": -0.06626236598202721, + "nauc_ndcg_at_5_diff1": 0.9345275765844611, + "nauc_ndcg_at_5_max": 0.5874513007198465, + "nauc_ndcg_at_5_std": -0.06589892538427641, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.8978758169934562, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.9673202614378978, + "nauc_precision_at_10_diff1": 0.9291938997821384, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 0.9615624027388776, + "nauc_precision_at_1_diff1": 0.9384664948453604, + "nauc_precision_at_1_max": 0.4803448699067248, + "nauc_precision_at_1_std": -0.154452012763868, + "nauc_precision_at_20_diff1": 0.939096850861544, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.9881164587046756, + "nauc_precision_at_3_diff1": 0.927760577915378, + "nauc_precision_at_3_max": 0.8183694530443756, + "nauc_precision_at_3_std": 0.11475584385801363, + "nauc_precision_at_5_diff1": 0.9297385620915051, + "nauc_precision_at_5_max": 0.9021319638966733, + "nauc_precision_at_5_std": 0.2836912542794903, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.8978758169934502, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.967320261437891, + "nauc_recall_at_10_diff1": 0.9291938997821416, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": 0.9615624027388804, + "nauc_recall_at_1_diff1": 0.9384664948453604, + "nauc_recall_at_1_max": 0.4803448699067248, + "nauc_recall_at_1_std": -0.154452012763868, + "nauc_recall_at_20_diff1": 0.9390968508615612, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.9881164587046992, + "nauc_recall_at_3_diff1": 0.9277605779153777, + "nauc_recall_at_3_max": 0.818369453044375, + "nauc_recall_at_3_std": 0.11475584385801059, + "nauc_recall_at_5_diff1": 0.9297385620915044, + "nauc_recall_at_5_max": 0.9021319638966716, + "nauc_recall_at_5_std": 0.2836912542794931, + "ndcg_at_1": 0.84, + "ndcg_at_10": 0.91689, + "ndcg_at_100": 0.91917, + "ndcg_at_1000": 0.91973, + "ndcg_at_20": 0.91794, + "ndcg_at_3": 0.90093, + "ndcg_at_5": 0.91203, + "precision_at_1": 0.84, + "precision_at_10": 0.0985, + "precision_at_100": 0.00996, + "precision_at_1000": 0.001, + "precision_at_20": 0.04945, + "precision_at_3": 0.31433, + "precision_at_5": 0.194, + "recall_at_1": 0.84, + "recall_at_10": 0.985, + "recall_at_100": 0.996, + "recall_at_1000": 1.0, + "recall_at_20": 0.989, + "recall_at_3": 0.943, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "CodeSearchNetRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CodeTransOceanContest.json b/results/voyageai__voyage-3/no_revision_available/CodeTransOceanContest.json new file mode 100644 index 000000000..075b151e2 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CodeTransOceanContest.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "20da4eb20a4b17300c0986ee148c90867a7f2a4d", + "evaluation_time": 10.410393953323364, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "python-Code", + "c++-Code" + ], + "main_score": 0.89919, + "map_at_1": 0.82805, + "map_at_10": 0.87795, + "map_at_100": 0.87929, + "map_at_1000": 0.87934, + "map_at_20": 0.87896, + "map_at_3": 0.86953, + "map_at_5": 0.87609, + "mrr_at_1": 0.8280542986425339, + "mrr_at_10": 0.877946563240681, + "mrr_at_100": 0.8792871893217914, + "mrr_at_1000": 0.8793359184599646, + "mrr_at_20": 0.8789612348435878, + "mrr_at_3": 0.8695324283559579, + "mrr_at_5": 0.8760935143288084, + "nauc_map_at_1000_diff1": 0.8810086207039382, + "nauc_map_at_1000_max": 0.6991786259903227, + "nauc_map_at_1000_std": -0.06835254468096472, + "nauc_map_at_100_diff1": 0.8809418354951114, + "nauc_map_at_100_max": 0.6993053358150821, + "nauc_map_at_100_std": -0.06790890480228688, + "nauc_map_at_10_diff1": 0.8821526330599068, + "nauc_map_at_10_max": 0.7011278711338229, + "nauc_map_at_10_std": -0.07449369999640539, + "nauc_map_at_1_diff1": 0.892552927388668, + "nauc_map_at_1_max": 0.6697831038463516, + "nauc_map_at_1_std": -0.0746010219792033, + "nauc_map_at_20_diff1": 0.8811219234192318, + "nauc_map_at_20_max": 0.6993168801696037, + "nauc_map_at_20_std": -0.07200143390122499, + "nauc_map_at_3_diff1": 0.8718282654258175, + "nauc_map_at_3_max": 0.6986519612443549, + "nauc_map_at_3_std": -0.03455430760312618, + "nauc_map_at_5_diff1": 0.8781578469704966, + "nauc_map_at_5_max": 0.7042472575967516, + "nauc_map_at_5_std": -0.06016639264080397, + "nauc_mrr_at_1000_diff1": 0.8810086207039382, + "nauc_mrr_at_1000_max": 0.6991786259903227, + "nauc_mrr_at_1000_std": -0.06835254468096472, + "nauc_mrr_at_100_diff1": 0.8809418354951114, + "nauc_mrr_at_100_max": 0.6993053358150821, + "nauc_mrr_at_100_std": -0.06790890480228688, + "nauc_mrr_at_10_diff1": 0.8821526330599068, + "nauc_mrr_at_10_max": 0.7011278711338229, + "nauc_mrr_at_10_std": -0.07449369999640539, + "nauc_mrr_at_1_diff1": 0.892552927388668, + "nauc_mrr_at_1_max": 0.6697831038463516, + "nauc_mrr_at_1_std": -0.0746010219792033, + "nauc_mrr_at_20_diff1": 0.8811219234192318, + "nauc_mrr_at_20_max": 0.6993168801696037, + "nauc_mrr_at_20_std": -0.07200143390122499, + "nauc_mrr_at_3_diff1": 0.8718282654258175, + "nauc_mrr_at_3_max": 0.6986519612443549, + "nauc_mrr_at_3_std": -0.03455430760312618, + "nauc_mrr_at_5_diff1": 0.8781578469704966, + "nauc_mrr_at_5_max": 0.7042472575967516, + "nauc_mrr_at_5_std": -0.06016639264080397, + "nauc_ndcg_at_1000_diff1": 0.879540178318482, + "nauc_ndcg_at_1000_max": 0.7050146173900038, + "nauc_ndcg_at_1000_std": -0.07088006023365873, + "nauc_ndcg_at_100_diff1": 0.8775713794463942, + "nauc_ndcg_at_100_max": 0.7087117420146529, + "nauc_ndcg_at_100_std": -0.05813108890954731, + "nauc_ndcg_at_10_diff1": 0.8839620393601927, + "nauc_ndcg_at_10_max": 0.7149517031021715, + "nauc_ndcg_at_10_std": -0.1065666174590199, + "nauc_ndcg_at_1_diff1": 0.892552927388668, + "nauc_ndcg_at_1_max": 0.6697831038463516, + "nauc_ndcg_at_1_std": -0.0746010219792033, + "nauc_ndcg_at_20_diff1": 0.8795907234536295, + "nauc_ndcg_at_20_max": 0.707555750187581, + "nauc_ndcg_at_20_std": -0.09765435995224017, + "nauc_ndcg_at_3_diff1": 0.8619259910242845, + "nauc_ndcg_at_3_max": 0.7112263759310588, + "nauc_ndcg_at_3_std": -0.01388032641742912, + "nauc_ndcg_at_5_diff1": 0.873332422073453, + "nauc_ndcg_at_5_max": 0.723369820235741, + "nauc_ndcg_at_5_std": -0.0649143059440735, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7224540783011153, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 0.934673366834198, + "nauc_precision_at_10_diff1": 0.914281861283817, + "nauc_precision_at_10_max": 0.836214712539855, + "nauc_precision_at_10_std": -0.4727791621469819, + "nauc_precision_at_1_diff1": 0.892552927388668, + "nauc_precision_at_1_max": 0.6697831038463516, + "nauc_precision_at_1_std": -0.0746010219792033, + "nauc_precision_at_20_diff1": 0.8628509780541069, + "nauc_precision_at_20_max": 0.7902048465964379, + "nauc_precision_at_20_std": -0.555682595744694, + "nauc_precision_at_3_diff1": 0.8151000138024483, + "nauc_precision_at_3_max": 0.7701515629247735, + "nauc_precision_at_3_std": 0.0845609393392571, + "nauc_precision_at_5_diff1": 0.8422981021138841, + "nauc_precision_at_5_max": 0.8571282879686837, + "nauc_precision_at_5_std": -0.11159283221442548, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7224540783011127, + "nauc_recall_at_100_max": 1.0, + "nauc_recall_at_100_std": 0.9346733668341689, + "nauc_recall_at_10_diff1": 0.9142818612838207, + "nauc_recall_at_10_max": 0.8362147125398569, + "nauc_recall_at_10_std": -0.4727791621469815, + "nauc_recall_at_1_diff1": 0.892552927388668, + "nauc_recall_at_1_max": 0.6697831038463516, + "nauc_recall_at_1_std": -0.0746010219792033, + "nauc_recall_at_20_diff1": 0.8628509780541135, + "nauc_recall_at_20_max": 0.7902048465964362, + "nauc_recall_at_20_std": -0.5556825957446879, + "nauc_recall_at_3_diff1": 0.8151000138024498, + "nauc_recall_at_3_max": 0.7701515629247748, + "nauc_recall_at_3_std": 0.08456093933925808, + "nauc_recall_at_5_diff1": 0.8422981021138861, + "nauc_recall_at_5_max": 0.8571282879686849, + "nauc_recall_at_5_std": -0.11159283221441775, + "ndcg_at_1": 0.82805, + "ndcg_at_10": 0.89919, + "ndcg_at_100": 0.90521, + "ndcg_at_1000": 0.90635, + "ndcg_at_20": 0.90271, + "ndcg_at_3": 0.88327, + "ndcg_at_5": 0.89476, + "precision_at_1": 0.82805, + "precision_at_10": 0.09638, + "precision_at_100": 0.00991, + "precision_at_1000": 0.001, + "precision_at_20": 0.04887, + "precision_at_3": 0.30769, + "precision_at_5": 0.19005, + "recall_at_1": 0.82805, + "recall_at_10": 0.9638, + "recall_at_100": 0.99095, + "recall_at_1000": 1.0, + "recall_at_20": 0.97738, + "recall_at_3": 0.92308, + "recall_at_5": 0.95023 + } + ] + }, + "task_name": "CodeTransOceanContest" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CodeTransOceanDL.json b/results/voyageai__voyage-3/no_revision_available/CodeTransOceanDL.json new file mode 100644 index 000000000..030e16e37 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CodeTransOceanDL.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "281562cb8a1265ab5c0824bfa6ddcd9b0a15618f", + "evaluation_time": 8.577436208724976, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "python-Code" + ], + "main_score": 0.33924, + "map_at_1": 0.07778, + "map_at_10": 0.20573, + "map_at_100": 0.22301, + "map_at_1000": 0.22301, + "map_at_20": 0.22301, + "map_at_3": 0.10556, + "map_at_5": 0.14361, + "mrr_at_1": 0.03333333333333333, + "mrr_at_10": 0.17942901234567915, + "mrr_at_100": 0.1958191037357705, + "mrr_at_1000": 0.1958191037357705, + "mrr_at_20": 0.1958191037357705, + "mrr_at_3": 0.075, + "mrr_at_5": 0.11694444444444445, + "nauc_map_at_1000_diff1": -0.10560406550292167, + "nauc_map_at_1000_max": -0.3456106844047046, + "nauc_map_at_1000_std": -0.16865734838639496, + "nauc_map_at_100_diff1": -0.10560406550292167, + "nauc_map_at_100_max": -0.3456106844047046, + "nauc_map_at_100_std": -0.16865734838639496, + "nauc_map_at_10_diff1": -0.09748885929668269, + "nauc_map_at_10_max": -0.3068496531243908, + "nauc_map_at_10_std": -0.15894602920873457, + "nauc_map_at_1_diff1": -0.2650734025834644, + "nauc_map_at_1_max": -0.4018551880102727, + "nauc_map_at_1_std": -0.1573076775652575, + "nauc_map_at_20_diff1": -0.10560406550292167, + "nauc_map_at_20_max": -0.3456106844047046, + "nauc_map_at_20_std": -0.16865734838639496, + "nauc_map_at_3_diff1": -0.14576402667342955, + "nauc_map_at_3_max": -0.42048941068826806, + "nauc_map_at_3_std": -0.2044391071064501, + "nauc_map_at_5_diff1": -0.07252254018158114, + "nauc_map_at_5_max": -0.3417960637078389, + "nauc_map_at_5_std": -0.1716706330359681, + "nauc_mrr_at_1000_diff1": 0.12519236544052675, + "nauc_mrr_at_1000_max": -0.255587058169204, + "nauc_mrr_at_1000_std": -0.2277007909959776, + "nauc_mrr_at_100_diff1": 0.12519236544052675, + "nauc_mrr_at_100_max": -0.255587058169204, + "nauc_mrr_at_100_std": -0.2277007909959776, + "nauc_mrr_at_10_diff1": 0.13207894114023971, + "nauc_mrr_at_10_max": -0.20726986482225027, + "nauc_mrr_at_10_std": -0.2017459343565723, + "nauc_mrr_at_1_diff1": -0.14971827206868804, + "nauc_mrr_at_1_max": -0.46158662015919877, + "nauc_mrr_at_1_std": -0.417315088095877, + "nauc_mrr_at_20_diff1": 0.12519236544052675, + "nauc_mrr_at_20_max": -0.255587058169204, + "nauc_mrr_at_20_std": -0.2277007909959776, + "nauc_mrr_at_3_diff1": 0.13914658729907442, + "nauc_mrr_at_3_max": -0.32558654972611867, + "nauc_mrr_at_3_std": -0.23689607981790495, + "nauc_mrr_at_5_diff1": 0.14206899925523372, + "nauc_mrr_at_5_max": -0.22531580869546164, + "nauc_mrr_at_5_std": -0.208291336561497, + "nauc_ndcg_at_1000_diff1": -0.09026408788771803, + "nauc_ndcg_at_1000_max": -0.3344375118642577, + "nauc_ndcg_at_1000_std": -0.16764177164807773, + "nauc_ndcg_at_100_diff1": -0.09026408788771803, + "nauc_ndcg_at_100_max": -0.3344375118642577, + "nauc_ndcg_at_100_std": -0.16764177164807773, + "nauc_ndcg_at_10_diff1": -0.06844335399180254, + "nauc_ndcg_at_10_max": -0.21955508334835103, + "nauc_ndcg_at_10_std": -0.13763854406362513, + "nauc_ndcg_at_1_diff1": -0.2650734025834644, + "nauc_ndcg_at_1_max": -0.4018551880102727, + "nauc_ndcg_at_1_std": -0.1573076775652575, + "nauc_ndcg_at_20_diff1": -0.09026408788771803, + "nauc_ndcg_at_20_max": -0.3344375118642577, + "nauc_ndcg_at_20_std": -0.16764177164807773, + "nauc_ndcg_at_3_diff1": -0.12080774735694882, + "nauc_ndcg_at_3_max": -0.4270969665958267, + "nauc_ndcg_at_3_std": -0.21974861263415366, + "nauc_ndcg_at_5_diff1": -0.013965730303239452, + "nauc_ndcg_at_5_max": -0.30979844424508735, + "nauc_ndcg_at_5_std": -0.17035062262828415, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": -0.0073953823953833075, + "nauc_precision_at_10_max": 0.08806471306471345, + "nauc_precision_at_10_std": -0.05559718059718038, + "nauc_precision_at_1_diff1": -0.2650734025834644, + "nauc_precision_at_1_max": -0.4018551880102727, + "nauc_precision_at_1_std": -0.1573076775652575, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_3_diff1": -0.0673991197123947, + "nauc_precision_at_3_max": -0.4419917347041631, + "nauc_precision_at_3_std": -0.25402905173089646, + "nauc_precision_at_5_diff1": 0.09210762064022981, + "nauc_precision_at_5_max": -0.2495520783564257, + "nauc_precision_at_5_std": -0.1669254658385089, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": -0.007395382395382432, + "nauc_recall_at_10_max": 0.08806471306471256, + "nauc_recall_at_10_std": -0.05559718059718019, + "nauc_recall_at_1_diff1": -0.2650734025834644, + "nauc_recall_at_1_max": -0.4018551880102727, + "nauc_recall_at_1_std": -0.1573076775652575, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": -0.06739911971239462, + "nauc_recall_at_3_max": -0.441991734704163, + "nauc_recall_at_3_std": -0.2540290517308964, + "nauc_recall_at_5_diff1": 0.0921076206402293, + "nauc_recall_at_5_max": -0.2495520783564264, + "nauc_recall_at_5_std": -0.16692546583850942, + "ndcg_at_1": 0.07778, + "ndcg_at_10": 0.33924, + "ndcg_at_100": 0.39399, + "ndcg_at_1000": 0.39399, + "ndcg_at_20": 0.39399, + "ndcg_at_3": 0.1168, + "ndcg_at_5": 0.18424, + "precision_at_1": 0.07778, + "precision_at_10": 0.08, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.05, + "precision_at_5": 0.06222, + "recall_at_1": 0.07778, + "recall_at_10": 0.8, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.15, + "recall_at_5": 0.31111 + } + ] + }, + "task_name": "CodeTransOceanDL" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/CosQA.json b/results/voyageai__voyage-3/no_revision_available/CosQA.json new file mode 100644 index 000000000..d27dd786a --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/CosQA.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "bc5efb7e9d437246ce393ed19d772e08e4a79535", + "evaluation_time": 97.1205484867096, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn", + "python-Code" + ], + "main_score": 0.28703, + "map_at_1": 0.132, + "map_at_10": 0.22186, + "map_at_100": 0.23634, + "map_at_1000": 0.23701, + "map_at_20": 0.23005, + "map_at_3": 0.176, + "map_at_5": 0.2022, + "mrr_at_1": 0.164, + "mrr_at_10": 0.23591507936507927, + "mrr_at_100": 0.25187893896442864, + "mrr_at_1000": 0.25254644694672274, + "mrr_at_20": 0.24516810965951824, + "mrr_at_3": 0.20399999999999993, + "mrr_at_5": 0.2138, + "nauc_map_at_1000_diff1": 0.22883261654529907, + "nauc_map_at_1000_max": 0.10395770467415337, + "nauc_map_at_1000_std": -0.179567611936642, + "nauc_map_at_100_diff1": 0.22857754174791958, + "nauc_map_at_100_max": 0.10446298540128005, + "nauc_map_at_100_std": -0.17896912072083673, + "nauc_map_at_10_diff1": 0.22532881702158433, + "nauc_map_at_10_max": 0.09625557989137662, + "nauc_map_at_10_std": -0.18681065895094948, + "nauc_map_at_1_diff1": 0.3592918316195942, + "nauc_map_at_1_max": 0.09158540797619358, + "nauc_map_at_1_std": -0.1811648112229986, + "nauc_map_at_20_diff1": 0.2224851142631941, + "nauc_map_at_20_max": 0.10355130675157224, + "nauc_map_at_20_std": -0.18291604357935132, + "nauc_map_at_3_diff1": 0.2607938036229648, + "nauc_map_at_3_max": 0.07940363668435517, + "nauc_map_at_3_std": -0.19546504676926987, + "nauc_map_at_5_diff1": 0.23296582822249673, + "nauc_map_at_5_max": 0.09519580769682656, + "nauc_map_at_5_std": -0.1923132876107064, + "nauc_mrr_at_1000_diff1": 0.2091009592534719, + "nauc_mrr_at_1000_max": 0.0394603685452002, + "nauc_mrr_at_1000_std": -0.23877313714611884, + "nauc_mrr_at_100_diff1": 0.20890887175676806, + "nauc_mrr_at_100_max": 0.04006649489480442, + "nauc_mrr_at_100_std": -0.23809367224561323, + "nauc_mrr_at_10_diff1": 0.20855614216971832, + "nauc_mrr_at_10_max": 0.031846293362517576, + "nauc_mrr_at_10_std": -0.24906855586284324, + "nauc_mrr_at_1_diff1": 0.24964756081931735, + "nauc_mrr_at_1_max": 0.022838812788830605, + "nauc_mrr_at_1_std": -0.2471409078230191, + "nauc_mrr_at_20_diff1": 0.20464706715001693, + "nauc_mrr_at_20_max": 0.03946566266594375, + "nauc_mrr_at_20_std": -0.2416659247607434, + "nauc_mrr_at_3_diff1": 0.23515108797081316, + "nauc_mrr_at_3_max": 0.013057977085785507, + "nauc_mrr_at_3_std": -0.2556227156159331, + "nauc_mrr_at_5_diff1": 0.2265359758860489, + "nauc_mrr_at_5_max": 0.018859865211682297, + "nauc_mrr_at_5_std": -0.25360314256092154, + "nauc_ndcg_at_1000_diff1": 0.20085508782749392, + "nauc_ndcg_at_1000_max": 0.12251343243272136, + "nauc_ndcg_at_1000_std": -0.159223691494969, + "nauc_ndcg_at_100_diff1": 0.19300247331632847, + "nauc_ndcg_at_100_max": 0.13481401886320638, + "nauc_ndcg_at_100_std": -0.14256245711701926, + "nauc_ndcg_at_10_diff1": 0.1709983441404966, + "nauc_ndcg_at_10_max": 0.10433701267146095, + "nauc_ndcg_at_10_std": -0.18290119621996828, + "nauc_ndcg_at_1_diff1": 0.3592918316195942, + "nauc_ndcg_at_1_max": 0.09158540797619358, + "nauc_ndcg_at_1_std": -0.1811648112229986, + "nauc_ndcg_at_20_diff1": 0.15746568670337255, + "nauc_ndcg_at_20_max": 0.1281845519698444, + "nauc_ndcg_at_20_std": -0.16760676144069886, + "nauc_ndcg_at_3_diff1": 0.23369402983850923, + "nauc_ndcg_at_3_max": 0.07440837789401826, + "nauc_ndcg_at_3_std": -0.20062610637774284, + "nauc_ndcg_at_5_diff1": 0.1888921117707364, + "nauc_ndcg_at_5_max": 0.1017765314613395, + "nauc_ndcg_at_5_std": -0.193807538118644, + "nauc_precision_at_1000_diff1": 0.09114015976761518, + "nauc_precision_at_1000_max": 0.6649548708372202, + "nauc_precision_at_1000_std": 0.570028011204478, + "nauc_precision_at_100_diff1": 0.07049477742352975, + "nauc_precision_at_100_max": 0.35863025759374756, + "nauc_precision_at_100_std": 0.16633539212678267, + "nauc_precision_at_10_diff1": 0.04023146578082273, + "nauc_precision_at_10_max": 0.12601106058218917, + "nauc_precision_at_10_std": -0.17093966882341824, + "nauc_precision_at_1_diff1": 0.3592918316195942, + "nauc_precision_at_1_max": 0.09158540797619358, + "nauc_precision_at_1_std": -0.1811648112229986, + "nauc_precision_at_20_diff1": -0.029544538246325722, + "nauc_precision_at_20_max": 0.21070909335255575, + "nauc_precision_at_20_std": -0.1117579504221086, + "nauc_precision_at_3_diff1": 0.1690541828381433, + "nauc_precision_at_3_max": 0.06208944993431217, + "nauc_precision_at_3_std": -0.21323801306545026, + "nauc_precision_at_5_diff1": 0.08824738139897569, + "nauc_precision_at_5_max": 0.11893295959872695, + "nauc_precision_at_5_std": -0.1959577651794558, + "nauc_recall_at_1000_diff1": 0.09114015976760567, + "nauc_recall_at_1000_max": 0.664954870837225, + "nauc_recall_at_1000_std": 0.5700280112044809, + "nauc_recall_at_100_diff1": 0.07049477742352996, + "nauc_recall_at_100_max": 0.358630257593747, + "nauc_recall_at_100_std": 0.1663353921267828, + "nauc_recall_at_10_diff1": 0.04023146578082309, + "nauc_recall_at_10_max": 0.12601106058218903, + "nauc_recall_at_10_std": -0.17093966882341793, + "nauc_recall_at_1_diff1": 0.3592918316195942, + "nauc_recall_at_1_max": 0.09158540797619358, + "nauc_recall_at_1_std": -0.1811648112229986, + "nauc_recall_at_20_diff1": -0.029544538246325705, + "nauc_recall_at_20_max": 0.21070909335255547, + "nauc_recall_at_20_std": -0.11175795042210865, + "nauc_recall_at_3_diff1": 0.16905418283814305, + "nauc_recall_at_3_max": 0.06208944993431201, + "nauc_recall_at_3_std": -0.2132380130654505, + "nauc_recall_at_5_diff1": 0.08824738139897585, + "nauc_recall_at_5_max": 0.11893295959872698, + "nauc_recall_at_5_std": -0.1959577651794557, + "ndcg_at_1": 0.132, + "ndcg_at_10": 0.28703, + "ndcg_at_100": 0.36065, + "ndcg_at_1000": 0.37728, + "ndcg_at_20": 0.3162, + "ndcg_at_3": 0.19181, + "ndcg_at_5": 0.23889, + "precision_at_1": 0.132, + "precision_at_10": 0.0502, + "precision_at_100": 0.00854, + "precision_at_1000": 0.00098, + "precision_at_20": 0.0308, + "precision_at_3": 0.07933, + "precision_at_5": 0.0704, + "recall_at_1": 0.132, + "recall_at_10": 0.502, + "recall_at_100": 0.854, + "recall_at_1000": 0.982, + "recall_at_20": 0.616, + "recall_at_3": 0.238, + "recall_at_5": 0.352 + } + ] + }, + "task_name": "CosQA" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/GerDaLIRSmall.json b/results/voyageai__voyage-3/no_revision_available/GerDaLIRSmall.json new file mode 100644 index 000000000..94653c061 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/GerDaLIRSmall.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "48327de6ee192e9610f3069789719788957c7abd", + "evaluation_time": 741.2940034866333, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "deu-Latn" + ], + "main_score": 0.44719, + "map_at_1": 0.29689, + "map_at_10": 0.39688, + "map_at_100": 0.40573, + "map_at_1000": 0.40627, + "map_at_20": 0.40232, + "map_at_3": 0.36956, + "map_at_5": 0.38593, + "mrr_at_1": 0.3267124407389243, + "mrr_at_10": 0.4162314042831455, + "mrr_at_100": 0.4235993115980269, + "mrr_at_1000": 0.424049145576368, + "mrr_at_20": 0.42077210329586107, + "mrr_at_3": 0.3926761484387774, + "mrr_at_5": 0.40663724047735583, + "nauc_map_at_1000_diff1": 0.44755027063928154, + "nauc_map_at_1000_max": 0.360649641547951, + "nauc_map_at_1000_std": 0.044347977098334795, + "nauc_map_at_100_diff1": 0.4474494593900217, + "nauc_map_at_100_max": 0.3607448893794954, + "nauc_map_at_100_std": 0.044546254642694176, + "nauc_map_at_10_diff1": 0.4486847936410468, + "nauc_map_at_10_max": 0.3587207930429742, + "nauc_map_at_10_std": 0.03847708339416474, + "nauc_map_at_1_diff1": 0.5068168215867382, + "nauc_map_at_1_max": 0.31715665330483395, + "nauc_map_at_1_std": -0.0057371181880882674, + "nauc_map_at_20_diff1": 0.4479231011958772, + "nauc_map_at_20_max": 0.36054894682684124, + "nauc_map_at_20_std": 0.042561912833134784, + "nauc_map_at_3_diff1": 0.458295013143185, + "nauc_map_at_3_max": 0.3474040092790643, + "nauc_map_at_3_std": 0.02104320406487452, + "nauc_map_at_5_diff1": 0.45156545971029965, + "nauc_map_at_5_max": 0.35489143592111816, + "nauc_map_at_5_std": 0.03139903867849832, + "nauc_mrr_at_1000_diff1": 0.45104264616417605, + "nauc_mrr_at_1000_max": 0.37313017935001774, + "nauc_mrr_at_1000_std": 0.05609030709475238, + "nauc_mrr_at_100_diff1": 0.45095203737211376, + "nauc_mrr_at_100_max": 0.3732315844219457, + "nauc_mrr_at_100_std": 0.05628880962411886, + "nauc_mrr_at_10_diff1": 0.451236493237472, + "nauc_mrr_at_10_max": 0.37249237297664384, + "nauc_mrr_at_10_std": 0.05231648456748863, + "nauc_mrr_at_1_diff1": 0.5064422145063561, + "nauc_mrr_at_1_max": 0.34782253476727254, + "nauc_mrr_at_1_std": 0.01139350963723932, + "nauc_mrr_at_20_diff1": 0.4511622629854407, + "nauc_mrr_at_20_max": 0.37342577388783027, + "nauc_mrr_at_20_std": 0.05517468316351196, + "nauc_mrr_at_3_diff1": 0.4600196358966202, + "nauc_mrr_at_3_max": 0.36612243441548875, + "nauc_mrr_at_3_std": 0.03846602340069412, + "nauc_mrr_at_5_diff1": 0.4542359569422018, + "nauc_mrr_at_5_max": 0.3706340982464388, + "nauc_mrr_at_5_std": 0.04693612524238818, + "nauc_ndcg_at_1000_diff1": 0.4238691851655385, + "nauc_ndcg_at_1000_max": 0.37677014715444446, + "nauc_ndcg_at_1000_std": 0.08340453043666972, + "nauc_ndcg_at_100_diff1": 0.4201751670386765, + "nauc_ndcg_at_100_max": 0.3801961318281692, + "nauc_ndcg_at_100_std": 0.0915242280751126, + "nauc_ndcg_at_10_diff1": 0.42552388569795746, + "nauc_ndcg_at_10_max": 0.37516394991031665, + "nauc_ndcg_at_10_std": 0.06630554064853446, + "nauc_ndcg_at_1_diff1": 0.5064422145063561, + "nauc_ndcg_at_1_max": 0.34782253476727254, + "nauc_ndcg_at_1_std": 0.01139350963723932, + "nauc_ndcg_at_20_diff1": 0.42273403000477605, + "nauc_ndcg_at_20_max": 0.38023407899735934, + "nauc_ndcg_at_20_std": 0.07991222285108444, + "nauc_ndcg_at_3_diff1": 0.4446886520964159, + "nauc_ndcg_at_3_max": 0.35779312245042105, + "nauc_ndcg_at_3_std": 0.033764984860942274, + "nauc_ndcg_at_5_diff1": 0.4332865606736024, + "nauc_ndcg_at_5_max": 0.3680801472105205, + "nauc_ndcg_at_5_std": 0.05052407750550954, + "nauc_precision_at_1000_diff1": 0.02852527558614205, + "nauc_precision_at_1000_max": 0.2362377581206244, + "nauc_precision_at_1000_std": 0.20923060897332146, + "nauc_precision_at_100_diff1": 0.10327393765132804, + "nauc_precision_at_100_max": 0.3239187081092362, + "nauc_precision_at_100_std": 0.26420333892879255, + "nauc_precision_at_10_diff1": 0.24819851671497353, + "nauc_precision_at_10_max": 0.3885806060322513, + "nauc_precision_at_10_std": 0.1612370362613409, + "nauc_precision_at_1_diff1": 0.5064422145063561, + "nauc_precision_at_1_max": 0.34782253476727254, + "nauc_precision_at_1_std": 0.01139350963723932, + "nauc_precision_at_20_diff1": 0.19596151483430677, + "nauc_precision_at_20_max": 0.3784159490988987, + "nauc_precision_at_20_std": 0.20865628830680646, + "nauc_precision_at_3_diff1": 0.3546679168967792, + "nauc_precision_at_3_max": 0.38190898332153445, + "nauc_precision_at_3_std": 0.0755700073057041, + "nauc_precision_at_5_diff1": 0.30360300095837117, + "nauc_precision_at_5_max": 0.3909200980153557, + "nauc_precision_at_5_std": 0.11570772300875601, + "nauc_recall_at_1000_diff1": 0.2327228626454581, + "nauc_recall_at_1000_max": 0.4169077463179734, + "nauc_recall_at_1000_std": 0.3468483855026672, + "nauc_recall_at_100_diff1": 0.26734047785131093, + "nauc_recall_at_100_max": 0.43818612705898946, + "nauc_recall_at_100_std": 0.3323945918992977, + "nauc_recall_at_10_diff1": 0.34137756894387405, + "nauc_recall_at_10_max": 0.4048555376890018, + "nauc_recall_at_10_std": 0.14161328587219077, + "nauc_recall_at_1_diff1": 0.5068168215867382, + "nauc_recall_at_1_max": 0.31715665330483395, + "nauc_recall_at_1_std": -0.0057371181880882674, + "nauc_recall_at_20_diff1": 0.3181245010617024, + "nauc_recall_at_20_max": 0.42580304140145664, + "nauc_recall_at_20_std": 0.20671287956642018, + "nauc_recall_at_3_diff1": 0.3996361458281994, + "nauc_recall_at_3_max": 0.3612430005458607, + "nauc_recall_at_3_std": 0.05117513035703744, + "nauc_recall_at_5_diff1": 0.37022362830313726, + "nauc_recall_at_5_max": 0.3837576259948542, + "nauc_recall_at_5_std": 0.09036091809645797, + "ndcg_at_1": 0.32671, + "ndcg_at_10": 0.44719, + "ndcg_at_100": 0.48809, + "ndcg_at_1000": 0.50258, + "ndcg_at_20": 0.46537, + "ndcg_at_3": 0.39562, + "ndcg_at_5": 0.42251, + "precision_at_1": 0.32671, + "precision_at_10": 0.06755, + "precision_at_100": 0.00902, + "precision_at_1000": 0.00103, + "precision_at_20": 0.038, + "precision_at_3": 0.1692, + "precision_at_5": 0.11707, + "recall_at_1": 0.29689, + "recall_at_10": 0.58256, + "recall_at_100": 0.7686, + "recall_at_1000": 0.88205, + "recall_at_20": 0.65104, + "recall_at_3": 0.44651, + "recall_at_5": 0.51007 + } + ] + }, + "task_name": "GerDaLIRSmall" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LEMBNarrativeQARetrieval.json b/results/voyageai__voyage-3/no_revision_available/LEMBNarrativeQARetrieval.json new file mode 100644 index 000000000..a5d1459f3 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LEMBNarrativeQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 138.1075141429901, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.5412, + "map_at_1": 0.43296, + "map_at_10": 0.50339, + "map_at_100": 0.51196, + "map_at_1000": 0.51261, + "map_at_20": 0.50807, + "map_at_3": 0.4833, + "map_at_5": 0.49451, + "mrr_at_1": 0.43296009187482054, + "mrr_at_10": 0.503386137049645, + "mrr_at_100": 0.5119570893933697, + "mrr_at_1000": 0.5126052264872621, + "mrr_at_20": 0.50806730984516, + "mrr_at_3": 0.4832998373050052, + "mrr_at_5": 0.4945066513541941, + "nauc_map_at_1000_diff1": 0.6544244456744245, + "nauc_map_at_1000_max": 0.5353898465402269, + "nauc_map_at_1000_std": 0.13887367965567982, + "nauc_map_at_100_diff1": 0.6540697944614651, + "nauc_map_at_100_max": 0.5352400111222111, + "nauc_map_at_100_std": 0.13915214651209176, + "nauc_map_at_10_diff1": 0.6536997187220741, + "nauc_map_at_10_max": 0.535781379875458, + "nauc_map_at_10_std": 0.13812172008338883, + "nauc_map_at_1_diff1": 0.7060458331768685, + "nauc_map_at_1_max": 0.5425189908965051, + "nauc_map_at_1_std": 0.1220604849957128, + "nauc_map_at_20_diff1": 0.653281785841381, + "nauc_map_at_20_max": 0.535299043751331, + "nauc_map_at_20_std": 0.13922045667962749, + "nauc_map_at_3_diff1": 0.6644347789600263, + "nauc_map_at_3_max": 0.5408299622685071, + "nauc_map_at_3_std": 0.13140975815933298, + "nauc_map_at_5_diff1": 0.6563197441997936, + "nauc_map_at_5_max": 0.5374637509555665, + "nauc_map_at_5_std": 0.13503847782330194, + "nauc_mrr_at_1000_diff1": 0.6544244456744245, + "nauc_mrr_at_1000_max": 0.5353898465402269, + "nauc_mrr_at_1000_std": 0.13887367965567982, + "nauc_mrr_at_100_diff1": 0.6540697944614651, + "nauc_mrr_at_100_max": 0.5352400111222111, + "nauc_mrr_at_100_std": 0.13915214651209176, + "nauc_mrr_at_10_diff1": 0.6536997187220741, + "nauc_mrr_at_10_max": 0.535781379875458, + "nauc_mrr_at_10_std": 0.13812172008338883, + "nauc_mrr_at_1_diff1": 0.7060458331768685, + "nauc_mrr_at_1_max": 0.5425189908965051, + "nauc_mrr_at_1_std": 0.1220604849957128, + "nauc_mrr_at_20_diff1": 0.653281785841381, + "nauc_mrr_at_20_max": 0.535299043751331, + "nauc_mrr_at_20_std": 0.13922045667962749, + "nauc_mrr_at_3_diff1": 0.6644347789600263, + "nauc_mrr_at_3_max": 0.5408299622685071, + "nauc_mrr_at_3_std": 0.13140975815933298, + "nauc_mrr_at_5_diff1": 0.6563197441997936, + "nauc_mrr_at_5_max": 0.5374637509555665, + "nauc_mrr_at_5_std": 0.13503847782330194, + "nauc_ndcg_at_1000_diff1": 0.6416881567080459, + "nauc_ndcg_at_1000_max": 0.5318218440385502, + "nauc_ndcg_at_1000_std": 0.1459885475761038, + "nauc_ndcg_at_100_diff1": 0.6331784909818423, + "nauc_ndcg_at_100_max": 0.5283004519366438, + "nauc_ndcg_at_100_std": 0.1533672014048165, + "nauc_ndcg_at_10_diff1": 0.629028428522794, + "nauc_ndcg_at_10_max": 0.5299200361842175, + "nauc_ndcg_at_10_std": 0.14946430395016527, + "nauc_ndcg_at_1_diff1": 0.7060458331768685, + "nauc_ndcg_at_1_max": 0.5425189908965051, + "nauc_ndcg_at_1_std": 0.1220604849957128, + "nauc_ndcg_at_20_diff1": 0.6269405485260062, + "nauc_ndcg_at_20_max": 0.5280942754222548, + "nauc_ndcg_at_20_std": 0.15376543044121802, + "nauc_ndcg_at_3_diff1": 0.6514533302414504, + "nauc_ndcg_at_3_max": 0.5404778549757212, + "nauc_ndcg_at_3_std": 0.13475582737910885, + "nauc_ndcg_at_5_diff1": 0.6362803322722281, + "nauc_ndcg_at_5_max": 0.5340646712895163, + "nauc_ndcg_at_5_std": 0.14127914364212166, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.47200614133383706, + "nauc_precision_at_100_max": 0.4622688026197153, + "nauc_precision_at_100_std": 0.2958793163461683, + "nauc_precision_at_10_diff1": 0.536956075708235, + "nauc_precision_at_10_max": 0.5064390129591846, + "nauc_precision_at_10_std": 0.194373741798724, + "nauc_precision_at_1_diff1": 0.7060458331768685, + "nauc_precision_at_1_max": 0.5425189908965051, + "nauc_precision_at_1_std": 0.1220604849957128, + "nauc_precision_at_20_diff1": 0.5105827939357855, + "nauc_precision_at_20_max": 0.4933316837720615, + "nauc_precision_at_20_std": 0.22394894345628347, + "nauc_precision_at_3_diff1": 0.6125752939016746, + "nauc_precision_at_3_max": 0.5394833147046263, + "nauc_precision_at_3_std": 0.14492426336142183, + "nauc_precision_at_5_diff1": 0.5712051916514754, + "nauc_precision_at_5_max": 0.5222877040345416, + "nauc_precision_at_5_std": 0.16183752413075383, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4720061413338389, + "nauc_recall_at_100_max": 0.46226880261971703, + "nauc_recall_at_100_std": 0.2958793163461718, + "nauc_recall_at_10_diff1": 0.536956075708235, + "nauc_recall_at_10_max": 0.5064390129591844, + "nauc_recall_at_10_std": 0.19437374179872471, + "nauc_recall_at_1_diff1": 0.7060458331768685, + "nauc_recall_at_1_max": 0.5425189908965051, + "nauc_recall_at_1_std": 0.1220604849957128, + "nauc_recall_at_20_diff1": 0.5105827939357848, + "nauc_recall_at_20_max": 0.49333168377206166, + "nauc_recall_at_20_std": 0.22394894345628388, + "nauc_recall_at_3_diff1": 0.6125752939016749, + "nauc_recall_at_3_max": 0.5394833147046261, + "nauc_recall_at_3_std": 0.14492426336142164, + "nauc_recall_at_5_diff1": 0.5712051916514757, + "nauc_recall_at_5_max": 0.5222877040345418, + "nauc_recall_at_5_std": 0.16183752413075395, + "ndcg_at_1": 0.43296, + "ndcg_at_10": 0.5412, + "ndcg_at_100": 0.58756, + "ndcg_at_1000": 0.60233, + "ndcg_at_20": 0.55827, + "ndcg_at_3": 0.49958, + "ndcg_at_5": 0.51967, + "precision_at_1": 0.43296, + "precision_at_10": 0.06618, + "precision_at_100": 0.0089, + "precision_at_1000": 0.001, + "precision_at_20": 0.03647, + "precision_at_3": 0.18222, + "precision_at_5": 0.11904, + "recall_at_1": 0.43296, + "recall_at_10": 0.66179, + "recall_at_100": 0.88965, + "recall_at_1000": 1.0, + "recall_at_20": 0.72935, + "recall_at_3": 0.54666, + "recall_at_5": 0.59518 + } + ] + }, + "task_name": "LEMBNarrativeQARetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LEMBNeedleRetrieval.json b/results/voyageai__voyage-3/no_revision_available/LEMBNeedleRetrieval.json new file mode 100644 index 000000000..43e9ac5af --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LEMBNeedleRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 72.97698020935059, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test_1024": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.74, + "map_at_1": 0.74, + "map_at_10": 0.864, + "map_at_100": 0.864, + "map_at_1000": 0.864, + "map_at_20": 0.864, + "map_at_3": 0.86, + "map_at_5": 0.864, + "mrr_at_1": 0.74, + "mrr_at_10": 0.8640000000000001, + "mrr_at_100": 0.8640000000000001, + "mrr_at_1000": 0.8640000000000001, + "mrr_at_20": 0.8640000000000001, + "mrr_at_3": 0.86, + "mrr_at_5": 0.8640000000000001, + "nauc_map_at_1000_diff1": 0.859632624430552, + "nauc_map_at_1000_max": 0.1667604504705877, + "nauc_map_at_1000_std": -0.28590699216070226, + "nauc_map_at_100_diff1": 0.859632624430552, + "nauc_map_at_100_max": 0.1667604504705877, + "nauc_map_at_100_std": -0.28590699216070226, + "nauc_map_at_10_diff1": 0.859632624430552, + "nauc_map_at_10_max": 0.1667604504705877, + "nauc_map_at_10_std": -0.28590699216070226, + "nauc_map_at_1_diff1": 0.8667937228822835, + "nauc_map_at_1_max": 0.2354730140707999, + "nauc_map_at_1_std": -0.2101293540407931, + "nauc_map_at_20_diff1": 0.859632624430552, + "nauc_map_at_20_max": 0.1667604504705877, + "nauc_map_at_20_std": -0.28590699216070226, + "nauc_map_at_3_diff1": 0.8552400850651054, + "nauc_map_at_3_max": 0.12461291646457537, + "nauc_map_at_3_std": -0.33238816550386135, + "nauc_map_at_5_diff1": 0.859632624430552, + "nauc_map_at_5_max": 0.1667604504705877, + "nauc_map_at_5_std": -0.28590699216070226, + "nauc_mrr_at_1000_diff1": 0.859632624430552, + "nauc_mrr_at_1000_max": 0.1667604504705877, + "nauc_mrr_at_1000_std": -0.28590699216070226, + "nauc_mrr_at_100_diff1": 0.859632624430552, + "nauc_mrr_at_100_max": 0.1667604504705877, + "nauc_mrr_at_100_std": -0.28590699216070226, + "nauc_mrr_at_10_diff1": 0.859632624430552, + "nauc_mrr_at_10_max": 0.1667604504705877, + "nauc_mrr_at_10_std": -0.28590699216070226, + "nauc_mrr_at_1_diff1": 0.8667937228822835, + "nauc_mrr_at_1_max": 0.2354730140707999, + "nauc_mrr_at_1_std": -0.2101293540407931, + "nauc_mrr_at_20_diff1": 0.859632624430552, + "nauc_mrr_at_20_max": 0.1667604504705877, + "nauc_mrr_at_20_std": -0.28590699216070226, + "nauc_mrr_at_3_diff1": 0.8552400850651054, + "nauc_mrr_at_3_max": 0.12461291646457537, + "nauc_mrr_at_3_std": -0.33238816550386135, + "nauc_mrr_at_5_diff1": 0.859632624430552, + "nauc_mrr_at_5_max": 0.1667604504705877, + "nauc_mrr_at_5_std": -0.28590699216070226, + "nauc_ndcg_at_1000_diff1": 0.8589404116700877, + "nauc_ndcg_at_1000_max": 0.1601184927250089, + "nauc_ndcg_at_1000_std": -0.2932318807120967, + "nauc_ndcg_at_100_diff1": 0.8589404116700877, + "nauc_ndcg_at_100_max": 0.1601184927250089, + "nauc_ndcg_at_100_std": -0.2932318807120967, + "nauc_ndcg_at_10_diff1": 0.8589404116700877, + "nauc_ndcg_at_10_max": 0.1601184927250089, + "nauc_ndcg_at_10_std": -0.2932318807120967, + "nauc_ndcg_at_1_diff1": 0.8667937228822835, + "nauc_ndcg_at_1_max": 0.2354730140707999, + "nauc_ndcg_at_1_std": -0.2101293540407931, + "nauc_ndcg_at_20_diff1": 0.8589404116700877, + "nauc_ndcg_at_20_max": 0.1601184927250089, + "nauc_ndcg_at_20_std": -0.2932318807120967, + "nauc_ndcg_at_3_diff1": 0.8481024725195574, + "nauc_ndcg_at_3_max": 0.05612570611614793, + "nauc_ndcg_at_3_std": -0.40791727939081895, + "nauc_ndcg_at_5_diff1": 0.8589404116700877, + "nauc_ndcg_at_5_max": 0.1601184927250089, + "nauc_ndcg_at_5_std": -0.2932318807120967, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.8667937228822835, + "nauc_precision_at_1_max": 0.2354730140707999, + "nauc_precision_at_1_std": -0.2101293540407931, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.7222222222222265, + "nauc_precision_at_3_max": -1.1517273576096958, + "nauc_precision_at_3_std": -1.7399626517273423, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.8667937228822835, + "nauc_recall_at_1_max": 0.2354730140707999, + "nauc_recall_at_1_std": -0.2101293540407931, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.7222222222222248, + "nauc_recall_at_3_max": -1.1517273576097098, + "nauc_recall_at_3_std": -1.739962651727357, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.74, + "ndcg_at_10": 0.89916, + "ndcg_at_100": 0.89916, + "ndcg_at_1000": 0.89916, + "ndcg_at_20": 0.89916, + "ndcg_at_3": 0.89142, + "ndcg_at_5": 0.89916, + "precision_at_1": 0.74, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.32667, + "precision_at_5": 0.2, + "recall_at_1": 0.74, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.98, + "recall_at_5": 1.0 + } + ], + "test_16384": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.32, + "map_at_1": 0.32, + "map_at_10": 0.43367, + "map_at_100": 0.44369, + "map_at_1000": 0.44369, + "map_at_20": 0.4391, + "map_at_3": 0.38333, + "map_at_5": 0.40833, + "mrr_at_1": 0.32, + "mrr_at_10": 0.43366666666666653, + "mrr_at_100": 0.4436937196752784, + "mrr_at_1000": 0.4436937196752784, + "mrr_at_20": 0.4391008970389773, + "mrr_at_3": 0.3833333333333333, + "mrr_at_5": 0.4083333333333332, + "nauc_map_at_1000_diff1": 0.5633140886294585, + "nauc_map_at_1000_max": 0.1803388971938983, + "nauc_map_at_1000_std": 0.3038909615759363, + "nauc_map_at_100_diff1": 0.5633140886294585, + "nauc_map_at_100_max": 0.1803388971938983, + "nauc_map_at_100_std": 0.3038909615759363, + "nauc_map_at_10_diff1": 0.561808103779794, + "nauc_map_at_10_max": 0.184573150288687, + "nauc_map_at_10_std": 0.3014849867279635, + "nauc_map_at_1_diff1": 0.6215615969174781, + "nauc_map_at_1_max": 0.1831049983945198, + "nauc_map_at_1_std": 0.2118430910842343, + "nauc_map_at_20_diff1": 0.5612642838704813, + "nauc_map_at_20_max": 0.1805451342615401, + "nauc_map_at_20_std": 0.3105230230114746, + "nauc_map_at_3_diff1": 0.5629797835049275, + "nauc_map_at_3_max": 0.1858286117418815, + "nauc_map_at_3_std": 0.3068233426662847, + "nauc_map_at_5_diff1": 0.5573123240527457, + "nauc_map_at_5_max": 0.18554985470089705, + "nauc_map_at_5_std": 0.2949659469980888, + "nauc_mrr_at_1000_diff1": 0.5633140886294585, + "nauc_mrr_at_1000_max": 0.1803388971938983, + "nauc_mrr_at_1000_std": 0.3038909615759363, + "nauc_mrr_at_100_diff1": 0.5633140886294585, + "nauc_mrr_at_100_max": 0.1803388971938983, + "nauc_mrr_at_100_std": 0.3038909615759363, + "nauc_mrr_at_10_diff1": 0.561808103779794, + "nauc_mrr_at_10_max": 0.184573150288687, + "nauc_mrr_at_10_std": 0.3014849867279635, + "nauc_mrr_at_1_diff1": 0.6215615969174781, + "nauc_mrr_at_1_max": 0.1831049983945198, + "nauc_mrr_at_1_std": 0.2118430910842343, + "nauc_mrr_at_20_diff1": 0.5612642838704813, + "nauc_mrr_at_20_max": 0.1805451342615401, + "nauc_mrr_at_20_std": 0.3105230230114746, + "nauc_mrr_at_3_diff1": 0.5629797835049275, + "nauc_mrr_at_3_max": 0.1858286117418815, + "nauc_mrr_at_3_std": 0.3068233426662847, + "nauc_mrr_at_5_diff1": 0.5573123240527457, + "nauc_mrr_at_5_max": 0.18554985470089705, + "nauc_mrr_at_5_std": 0.2949659469980888, + "nauc_ndcg_at_1000_diff1": 0.5574328051968741, + "nauc_ndcg_at_1000_max": 0.1820268825744041, + "nauc_ndcg_at_1000_std": 0.31951346466182545, + "nauc_ndcg_at_100_diff1": 0.5574328051968741, + "nauc_ndcg_at_100_max": 0.1820268825744041, + "nauc_ndcg_at_100_std": 0.31951346466182545, + "nauc_ndcg_at_10_diff1": 0.5482788971118266, + "nauc_ndcg_at_10_max": 0.19692003816881948, + "nauc_ndcg_at_10_std": 0.3242457312209489, + "nauc_ndcg_at_1_diff1": 0.6215615969174781, + "nauc_ndcg_at_1_max": 0.1831049983945198, + "nauc_ndcg_at_1_std": 0.2118430910842343, + "nauc_ndcg_at_20_diff1": 0.5453709374240684, + "nauc_ndcg_at_20_max": 0.18459805652843983, + "nauc_ndcg_at_20_std": 0.3636041152432253, + "nauc_ndcg_at_3_diff1": 0.5493840575437549, + "nauc_ndcg_at_3_max": 0.1944386447282744, + "nauc_ndcg_at_3_std": 0.3394078319892048, + "nauc_ndcg_at_5_diff1": 0.5397476465243035, + "nauc_ndcg_at_5_max": 0.19639433807550488, + "nauc_ndcg_at_5_std": 0.3203374185980016, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.48945147679324874, + "nauc_precision_at_10_max": 0.26549821486530284, + "nauc_precision_at_10_std": 0.40988547317661245, + "nauc_precision_at_1_diff1": 0.6215615969174781, + "nauc_precision_at_1_max": 0.1831049983945198, + "nauc_precision_at_1_std": 0.2118430910842343, + "nauc_precision_at_20_diff1": 0.4368851323264526, + "nauc_precision_at_20_max": 0.21283081612858615, + "nauc_precision_at_20_std": 0.7852293196619116, + "nauc_precision_at_3_diff1": 0.5118919345089483, + "nauc_precision_at_3_max": 0.22154476886284222, + "nauc_precision_at_3_std": 0.4337073927097842, + "nauc_precision_at_5_diff1": 0.4862292132222768, + "nauc_precision_at_5_max": 0.2346402762863857, + "nauc_precision_at_5_std": 0.3981193951882055, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.4894514767932486, + "nauc_recall_at_10_max": 0.2654982148653032, + "nauc_recall_at_10_std": 0.40988547317661195, + "nauc_recall_at_1_diff1": 0.6215615969174781, + "nauc_recall_at_1_max": 0.1831049983945198, + "nauc_recall_at_1_std": 0.2118430910842343, + "nauc_recall_at_20_diff1": 0.43688513232645076, + "nauc_recall_at_20_max": 0.21283081612858523, + "nauc_recall_at_20_std": 0.7852293196619083, + "nauc_recall_at_3_diff1": 0.5118919345089483, + "nauc_recall_at_3_max": 0.2215447688628417, + "nauc_recall_at_3_std": 0.4337073927097843, + "nauc_recall_at_5_diff1": 0.4862292132222771, + "nauc_recall_at_5_max": 0.2346402762863857, + "nauc_recall_at_5_std": 0.3981193951882054, + "ndcg_at_1": 0.32, + "ndcg_at_10": 0.51367, + "ndcg_at_100": 0.56175, + "ndcg_at_1000": 0.56175, + "ndcg_at_20": 0.53375, + "ndcg_at_3": 0.40309, + "ndcg_at_5": 0.45039, + "precision_at_1": 0.32, + "precision_at_10": 0.078, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.043, + "precision_at_3": 0.15333, + "precision_at_5": 0.116, + "recall_at_1": 0.32, + "recall_at_10": 0.78, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.86, + "recall_at_3": 0.46, + "recall_at_5": 0.58 + } + ], + "test_2048": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.64, + "map_at_1": 0.64, + "map_at_10": 0.74269, + "map_at_100": 0.74835, + "map_at_1000": 0.74835, + "map_at_20": 0.74748, + "map_at_3": 0.72333, + "map_at_5": 0.73733, + "mrr_at_1": 0.64, + "mrr_at_10": 0.7426904761904761, + "mrr_at_100": 0.7483452561930823, + "mrr_at_1000": 0.7483452561930823, + "mrr_at_20": 0.747475690975691, + "mrr_at_3": 0.7233333333333333, + "mrr_at_5": 0.7373333333333334, + "nauc_map_at_1000_diff1": 0.5313896018477882, + "nauc_map_at_1000_max": 0.17469683521428883, + "nauc_map_at_1000_std": 0.25885700234617204, + "nauc_map_at_100_diff1": 0.5313896018477882, + "nauc_map_at_100_max": 0.17469683521428883, + "nauc_map_at_100_std": 0.25885700234617204, + "nauc_map_at_10_diff1": 0.5289427343211895, + "nauc_map_at_10_max": 0.18727382464908998, + "nauc_map_at_10_std": 0.2652114574983357, + "nauc_map_at_1_diff1": 0.5275897304892068, + "nauc_map_at_1_max": 0.059522288925788334, + "nauc_map_at_1_std": 0.27723847234640403, + "nauc_map_at_20_diff1": 0.532163808869883, + "nauc_map_at_20_max": 0.17804508618040096, + "nauc_map_at_20_std": 0.2607368728123369, + "nauc_map_at_3_diff1": 0.5305174832544084, + "nauc_map_at_3_max": 0.17643300775204088, + "nauc_map_at_3_std": 0.2265084205465294, + "nauc_map_at_5_diff1": 0.528239587919223, + "nauc_map_at_5_max": 0.20132962434709742, + "nauc_map_at_5_std": 0.23850049926880396, + "nauc_mrr_at_1000_diff1": 0.5313896018477882, + "nauc_mrr_at_1000_max": 0.17469683521428883, + "nauc_mrr_at_1000_std": 0.25885700234617204, + "nauc_mrr_at_100_diff1": 0.5313896018477882, + "nauc_mrr_at_100_max": 0.17469683521428883, + "nauc_mrr_at_100_std": 0.25885700234617204, + "nauc_mrr_at_10_diff1": 0.5289427343211895, + "nauc_mrr_at_10_max": 0.18727382464908998, + "nauc_mrr_at_10_std": 0.2652114574983357, + "nauc_mrr_at_1_diff1": 0.5275897304892068, + "nauc_mrr_at_1_max": 0.059522288925788334, + "nauc_mrr_at_1_std": 0.27723847234640403, + "nauc_mrr_at_20_diff1": 0.532163808869883, + "nauc_mrr_at_20_max": 0.17804508618040096, + "nauc_mrr_at_20_std": 0.2607368728123369, + "nauc_mrr_at_3_diff1": 0.5305174832544084, + "nauc_mrr_at_3_max": 0.17643300775204088, + "nauc_mrr_at_3_std": 0.2265084205465294, + "nauc_mrr_at_5_diff1": 0.528239587919223, + "nauc_mrr_at_5_max": 0.20132962434709742, + "nauc_mrr_at_5_std": 0.23850049926880396, + "nauc_ndcg_at_1000_diff1": 0.5317440932429656, + "nauc_ndcg_at_1000_max": 0.19239170700251887, + "nauc_ndcg_at_1000_std": 0.26051291843274715, + "nauc_ndcg_at_100_diff1": 0.5317440932429656, + "nauc_ndcg_at_100_max": 0.19239170700251887, + "nauc_ndcg_at_100_std": 0.26051291843274715, + "nauc_ndcg_at_10_diff1": 0.5252562779629767, + "nauc_ndcg_at_10_max": 0.24464345656037173, + "nauc_ndcg_at_10_std": 0.29190714319738414, + "nauc_ndcg_at_1_diff1": 0.5275897304892068, + "nauc_ndcg_at_1_max": 0.059522288925788334, + "nauc_ndcg_at_1_std": 0.27723847234640403, + "nauc_ndcg_at_20_diff1": 0.5366519424356755, + "nauc_ndcg_at_20_max": 0.21320049869977722, + "nauc_ndcg_at_20_std": 0.2724092954394553, + "nauc_ndcg_at_3_diff1": 0.5295226717925118, + "nauc_ndcg_at_3_max": 0.22892198688593893, + "nauc_ndcg_at_3_std": 0.2001250784247086, + "nauc_ndcg_at_5_diff1": 0.5232072050972314, + "nauc_ndcg_at_5_max": 0.28032778683642995, + "nauc_ndcg_at_5_std": 0.21915152651801392, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.4919467787114846, + "nauc_precision_at_10_max": 0.6744864612511682, + "nauc_precision_at_10_std": 0.568277310924371, + "nauc_precision_at_1_diff1": 0.5275897304892068, + "nauc_precision_at_1_max": 0.059522288925788334, + "nauc_precision_at_1_std": 0.27723847234640403, + "nauc_precision_at_20_diff1": 0.7222222222222204, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 0.7222222222222204, + "nauc_precision_at_3_diff1": 0.5246172485956595, + "nauc_precision_at_3_max": 0.4411278775195497, + "nauc_precision_at_3_std": 0.0920806256195608, + "nauc_precision_at_5_diff1": 0.49069360483614577, + "nauc_precision_at_5_max": 0.7244670696786515, + "nauc_precision_at_5_std": 0.11175628380528234, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.4919467787114842, + "nauc_recall_at_10_max": 0.6744864612511672, + "nauc_recall_at_10_std": 0.5682773109243694, + "nauc_recall_at_1_diff1": 0.5275897304892068, + "nauc_recall_at_1_max": 0.059522288925788334, + "nauc_recall_at_1_std": 0.27723847234640403, + "nauc_recall_at_20_diff1": 0.7222222222222248, + "nauc_recall_at_20_max": 1.0, + "nauc_recall_at_20_std": 0.7222222222222248, + "nauc_recall_at_3_diff1": 0.5246172485956597, + "nauc_recall_at_3_max": 0.4411278775195506, + "nauc_recall_at_3_std": 0.09208062561956115, + "nauc_recall_at_5_diff1": 0.4906936048361431, + "nauc_recall_at_5_max": 0.7244670696786513, + "nauc_recall_at_5_std": 0.11175628380528144, + "ndcg_at_1": 0.64, + "ndcg_at_10": 0.78627, + "ndcg_at_100": 0.80658, + "ndcg_at_1000": 0.80658, + "ndcg_at_20": 0.80222, + "ndcg_at_3": 0.74833, + "ndcg_at_5": 0.77329, + "precision_at_1": 0.64, + "precision_at_10": 0.092, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.049, + "precision_at_3": 0.27333, + "precision_at_5": 0.176, + "recall_at_1": 0.64, + "recall_at_10": 0.92, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.98, + "recall_at_3": 0.82, + "recall_at_5": 0.88 + } + ], + "test_256": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86, + "map_at_1": 0.86, + "map_at_10": 0.93, + "map_at_100": 0.93, + "map_at_1000": 0.93, + "map_at_20": 0.93, + "map_at_3": 0.93, + "map_at_5": 0.93, + "mrr_at_1": 0.86, + "mrr_at_10": 0.93, + "mrr_at_100": 0.93, + "mrr_at_1000": 0.93, + "mrr_at_20": 0.93, + "mrr_at_3": 0.93, + "mrr_at_5": 0.93, + "nauc_map_at_1000_diff1": 0.8070527920188446, + "nauc_map_at_1000_max": 0.2654149923791023, + "nauc_map_at_1000_std": -0.04246917001524506, + "nauc_map_at_100_diff1": 0.8070527920188446, + "nauc_map_at_100_max": 0.2654149923791023, + "nauc_map_at_100_std": -0.04246917001524506, + "nauc_map_at_10_diff1": 0.8070527920188446, + "nauc_map_at_10_max": 0.2654149923791023, + "nauc_map_at_10_std": -0.04246917001524506, + "nauc_map_at_1_diff1": 0.8070527920188438, + "nauc_map_at_1_max": 0.265414992379104, + "nauc_map_at_1_std": -0.04246917001524114, + "nauc_map_at_20_diff1": 0.8070527920188446, + "nauc_map_at_20_max": 0.2654149923791023, + "nauc_map_at_20_std": -0.04246917001524506, + "nauc_map_at_3_diff1": 0.8070527920188446, + "nauc_map_at_3_max": 0.2654149923791023, + "nauc_map_at_3_std": -0.04246917001524506, + "nauc_map_at_5_diff1": 0.8070527920188446, + "nauc_map_at_5_max": 0.2654149923791023, + "nauc_map_at_5_std": -0.04246917001524506, + "nauc_mrr_at_1000_diff1": 0.8070527920188446, + "nauc_mrr_at_1000_max": 0.2654149923791023, + "nauc_mrr_at_1000_std": -0.04246917001524506, + "nauc_mrr_at_100_diff1": 0.8070527920188446, + "nauc_mrr_at_100_max": 0.2654149923791023, + "nauc_mrr_at_100_std": -0.04246917001524506, + "nauc_mrr_at_10_diff1": 0.8070527920188446, + "nauc_mrr_at_10_max": 0.2654149923791023, + "nauc_mrr_at_10_std": -0.04246917001524506, + "nauc_mrr_at_1_diff1": 0.8070527920188438, + "nauc_mrr_at_1_max": 0.265414992379104, + "nauc_mrr_at_1_std": -0.04246917001524114, + "nauc_mrr_at_20_diff1": 0.8070527920188446, + "nauc_mrr_at_20_max": 0.2654149923791023, + "nauc_mrr_at_20_std": -0.04246917001524506, + "nauc_mrr_at_3_diff1": 0.8070527920188446, + "nauc_mrr_at_3_max": 0.2654149923791023, + "nauc_mrr_at_3_std": -0.04246917001524506, + "nauc_mrr_at_5_diff1": 0.8070527920188446, + "nauc_mrr_at_5_max": 0.2654149923791023, + "nauc_mrr_at_5_std": -0.04246917001524506, + "nauc_ndcg_at_1000_diff1": 0.807052792018847, + "nauc_ndcg_at_1000_max": 0.2654149923791066, + "nauc_ndcg_at_1000_std": -0.04246917001523864, + "nauc_ndcg_at_100_diff1": 0.807052792018847, + "nauc_ndcg_at_100_max": 0.2654149923791066, + "nauc_ndcg_at_100_std": -0.04246917001523864, + "nauc_ndcg_at_10_diff1": 0.807052792018847, + "nauc_ndcg_at_10_max": 0.2654149923791066, + "nauc_ndcg_at_10_std": -0.04246917001523864, + "nauc_ndcg_at_1_diff1": 0.8070527920188438, + "nauc_ndcg_at_1_max": 0.265414992379104, + "nauc_ndcg_at_1_std": -0.04246917001524114, + "nauc_ndcg_at_20_diff1": 0.807052792018847, + "nauc_ndcg_at_20_max": 0.2654149923791066, + "nauc_ndcg_at_20_std": -0.04246917001523864, + "nauc_ndcg_at_3_diff1": 0.807052792018847, + "nauc_ndcg_at_3_max": 0.2654149923791066, + "nauc_ndcg_at_3_std": -0.04246917001523864, + "nauc_ndcg_at_5_diff1": 0.807052792018847, + "nauc_ndcg_at_5_max": 0.2654149923791066, + "nauc_ndcg_at_5_std": -0.04246917001523864, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.8070527920188438, + "nauc_precision_at_1_max": 0.265414992379104, + "nauc_precision_at_1_std": -0.04246917001524114, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.8070527920188438, + "nauc_recall_at_1_max": 0.265414992379104, + "nauc_recall_at_1_std": -0.04246917001524114, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.86, + "ndcg_at_10": 0.94833, + "ndcg_at_100": 0.94833, + "ndcg_at_1000": 0.94833, + "ndcg_at_20": 0.94833, + "ndcg_at_3": 0.94833, + "ndcg_at_5": 0.94833, + "precision_at_1": 0.86, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 0.86, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_32768": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.24, + "map_at_1": 0.24, + "map_at_10": 0.30489, + "map_at_100": 0.31648, + "map_at_1000": 0.31648, + "map_at_20": 0.30789, + "map_at_3": 0.27333, + "map_at_5": 0.28733, + "mrr_at_1": 0.24, + "mrr_at_10": 0.3048888888888888, + "mrr_at_100": 0.3158968097738347, + "mrr_at_1000": 0.3158968097738347, + "mrr_at_20": 0.3078888888888888, + "mrr_at_3": 0.2733333333333333, + "mrr_at_5": 0.28733333333333333, + "nauc_map_at_1000_diff1": 0.3315204815542972, + "nauc_map_at_1000_max": 0.0012693587415399253, + "nauc_map_at_1000_std": 0.3393836395306571, + "nauc_map_at_100_diff1": 0.3315204815542972, + "nauc_map_at_100_max": 0.0012693587415399253, + "nauc_map_at_100_std": 0.3393836395306571, + "nauc_map_at_10_diff1": 0.3339865595024787, + "nauc_map_at_10_max": -0.006794644985001, + "nauc_map_at_10_std": 0.3402202380426985, + "nauc_map_at_1_diff1": 0.40357656362329253, + "nauc_map_at_1_max": 0.03414809489575847, + "nauc_map_at_1_std": 0.31646896716990186, + "nauc_map_at_20_diff1": 0.32972758507322986, + "nauc_map_at_20_max": 0.0005681801110389538, + "nauc_map_at_20_std": 0.33814022457425635, + "nauc_map_at_3_diff1": 0.3475324404112125, + "nauc_map_at_3_max": 0.033707651855297276, + "nauc_map_at_3_std": 0.34695814783524936, + "nauc_map_at_5_diff1": 0.328126693284909, + "nauc_map_at_5_max": 0.010905814958296397, + "nauc_map_at_5_std": 0.3650587979196685, + "nauc_mrr_at_1000_diff1": 0.3333663049796035, + "nauc_mrr_at_1000_max": 0.0036758407601657664, + "nauc_mrr_at_1000_std": 0.3376286458198703, + "nauc_mrr_at_100_diff1": 0.3333663049796035, + "nauc_mrr_at_100_max": 0.0036758407601657664, + "nauc_mrr_at_100_std": 0.3376286458198703, + "nauc_mrr_at_10_diff1": 0.3339865595024787, + "nauc_mrr_at_10_max": -0.006794644985001, + "nauc_mrr_at_10_std": 0.3402202380426985, + "nauc_mrr_at_1_diff1": 0.40357656362329253, + "nauc_mrr_at_1_max": 0.03414809489575847, + "nauc_mrr_at_1_std": 0.31646896716990186, + "nauc_mrr_at_20_diff1": 0.32972758507322986, + "nauc_mrr_at_20_max": 0.0005681801110389538, + "nauc_mrr_at_20_std": 0.33814022457425635, + "nauc_mrr_at_3_diff1": 0.3475324404112125, + "nauc_mrr_at_3_max": 0.033707651855297276, + "nauc_mrr_at_3_std": 0.34695814783524936, + "nauc_mrr_at_5_diff1": 0.328126693284909, + "nauc_mrr_at_5_max": 0.010905814958296397, + "nauc_mrr_at_5_std": 0.3650587979196685, + "nauc_ndcg_at_1000_diff1": 0.31220212330050295, + "nauc_ndcg_at_1000_max": -0.009903427868577695, + "nauc_ndcg_at_1000_std": 0.3347080138418642, + "nauc_ndcg_at_100_diff1": 0.31220212330050295, + "nauc_ndcg_at_100_max": -0.009903427868577695, + "nauc_ndcg_at_100_std": 0.3347080138418642, + "nauc_ndcg_at_10_diff1": 0.30283515253541005, + "nauc_ndcg_at_10_max": -0.043917605919200416, + "nauc_ndcg_at_10_std": 0.32867391322539674, + "nauc_ndcg_at_1_diff1": 0.40357656362329253, + "nauc_ndcg_at_1_max": 0.03414809489575847, + "nauc_ndcg_at_1_std": 0.31646896716990186, + "nauc_ndcg_at_20_diff1": 0.2872647252081791, + "nauc_ndcg_at_20_max": -0.021408238442525825, + "nauc_ndcg_at_20_std": 0.32006475228143827, + "nauc_ndcg_at_3_diff1": 0.3272415419351875, + "nauc_ndcg_at_3_max": 0.03552778228104779, + "nauc_ndcg_at_3_std": 0.3517541674459727, + "nauc_ndcg_at_5_diff1": 0.2937421958174373, + "nauc_ndcg_at_5_max": -0.004754137973142522, + "nauc_ndcg_at_5_std": 0.3849413700603203, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.21212524983344416, + "nauc_precision_at_10_max": -0.1617854763491007, + "nauc_precision_at_10_std": 0.2822651565622916, + "nauc_precision_at_1_diff1": 0.40357656362329253, + "nauc_precision_at_1_max": 0.03414809489575847, + "nauc_precision_at_1_std": 0.31646896716990186, + "nauc_precision_at_20_diff1": 0.14812036342418924, + "nauc_precision_at_20_max": -0.08643827623572566, + "nauc_precision_at_20_std": 0.24497791114445264, + "nauc_precision_at_3_diff1": 0.27113882050733157, + "nauc_precision_at_3_max": 0.04104677298512263, + "nauc_precision_at_3_std": 0.36348068072353645, + "nauc_precision_at_5_diff1": 0.19730720245366853, + "nauc_precision_at_5_max": -0.05112676422426135, + "nauc_precision_at_5_std": 0.44173836196813326, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.21212524983344425, + "nauc_recall_at_10_max": -0.16178547634910045, + "nauc_recall_at_10_std": 0.2822651565622915, + "nauc_recall_at_1_diff1": 0.40357656362329253, + "nauc_recall_at_1_max": 0.03414809489575847, + "nauc_recall_at_1_std": 0.31646896716990186, + "nauc_recall_at_20_diff1": 0.14812036342418913, + "nauc_recall_at_20_max": -0.08643827623572585, + "nauc_recall_at_20_std": 0.24497791114445275, + "nauc_recall_at_3_diff1": 0.27113882050733185, + "nauc_recall_at_3_max": 0.04104677298512255, + "nauc_recall_at_3_std": 0.363480680723536, + "nauc_recall_at_5_diff1": 0.19730720245366873, + "nauc_recall_at_5_max": -0.05112676422426143, + "nauc_recall_at_5_std": 0.44173836196813354, + "ndcg_at_1": 0.24, + "ndcg_at_10": 0.3505, + "ndcg_at_100": 0.44, + "ndcg_at_1000": 0.44, + "ndcg_at_20": 0.3609, + "ndcg_at_3": 0.28524, + "ndcg_at_5": 0.3102, + "precision_at_1": 0.24, + "precision_at_10": 0.05, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.027, + "precision_at_3": 0.10667, + "precision_at_5": 0.076, + "recall_at_1": 0.24, + "recall_at_10": 0.5, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.54, + "recall_at_3": 0.32, + "recall_at_5": 0.38 + } + ], + "test_4096": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6, + "map_at_1": 0.6, + "map_at_10": 0.7435, + "map_at_100": 0.74642, + "map_at_1000": 0.74642, + "map_at_20": 0.74642, + "map_at_3": 0.73, + "map_at_5": 0.739, + "mrr_at_1": 0.6, + "mrr_at_10": 0.7434999999999999, + "mrr_at_100": 0.7464166666666666, + "mrr_at_1000": 0.7464166666666666, + "mrr_at_20": 0.7464166666666666, + "mrr_at_3": 0.73, + "mrr_at_5": 0.7390000000000001, + "nauc_map_at_1000_diff1": 0.371412513087309, + "nauc_map_at_1000_max": -0.16029611304603994, + "nauc_map_at_1000_std": -0.07331481462858515, + "nauc_map_at_100_diff1": 0.371412513087309, + "nauc_map_at_100_max": -0.16029611304603994, + "nauc_map_at_100_std": -0.07331481462858515, + "nauc_map_at_10_diff1": 0.3792963989041779, + "nauc_map_at_10_max": -0.1538526810410751, + "nauc_map_at_10_std": -0.06806906416469889, + "nauc_map_at_1_diff1": 0.29789254971801715, + "nauc_map_at_1_max": -0.21024042742653606, + "nauc_map_at_1_std": -0.08892846542000597, + "nauc_map_at_20_diff1": 0.371412513087309, + "nauc_map_at_20_max": -0.16029611304603994, + "nauc_map_at_20_std": -0.07331481462858515, + "nauc_map_at_3_diff1": 0.3995719391571772, + "nauc_map_at_3_max": -0.1369794697032672, + "nauc_map_at_3_std": -0.042619067409193086, + "nauc_map_at_5_diff1": 0.3822216754934584, + "nauc_map_at_5_max": -0.15172370630306403, + "nauc_map_at_5_std": -0.08345095195549086, + "nauc_mrr_at_1000_diff1": 0.371412513087309, + "nauc_mrr_at_1000_max": -0.16029611304603994, + "nauc_mrr_at_1000_std": -0.07331481462858515, + "nauc_mrr_at_100_diff1": 0.371412513087309, + "nauc_mrr_at_100_max": -0.16029611304603994, + "nauc_mrr_at_100_std": -0.07331481462858515, + "nauc_mrr_at_10_diff1": 0.3792963989041779, + "nauc_mrr_at_10_max": -0.1538526810410751, + "nauc_mrr_at_10_std": -0.06806906416469889, + "nauc_mrr_at_1_diff1": 0.29789254971801715, + "nauc_mrr_at_1_max": -0.21024042742653606, + "nauc_mrr_at_1_std": -0.08892846542000597, + "nauc_mrr_at_20_diff1": 0.371412513087309, + "nauc_mrr_at_20_max": -0.16029611304603994, + "nauc_mrr_at_20_std": -0.07331481462858515, + "nauc_mrr_at_3_diff1": 0.3995719391571772, + "nauc_mrr_at_3_max": -0.1369794697032672, + "nauc_mrr_at_3_std": -0.042619067409193086, + "nauc_mrr_at_5_diff1": 0.3822216754934584, + "nauc_mrr_at_5_max": -0.15172370630306403, + "nauc_mrr_at_5_std": -0.08345095195549086, + "nauc_ndcg_at_1000_diff1": 0.38204725946737783, + "nauc_ndcg_at_1000_max": -0.15232108136437872, + "nauc_ndcg_at_1000_std": -0.06976976094738162, + "nauc_ndcg_at_100_diff1": 0.38204725946737783, + "nauc_ndcg_at_100_max": -0.15232108136437872, + "nauc_ndcg_at_100_std": -0.06976976094738162, + "nauc_ndcg_at_10_diff1": 0.41578239152771046, + "nauc_ndcg_at_10_max": -0.12318459256361482, + "nauc_ndcg_at_10_std": -0.045643478376144464, + "nauc_ndcg_at_1_diff1": 0.29789254971801715, + "nauc_ndcg_at_1_max": -0.21024042742653606, + "nauc_ndcg_at_1_std": -0.08892846542000597, + "nauc_ndcg_at_20_diff1": 0.38204725946737783, + "nauc_ndcg_at_20_max": -0.15232108136437872, + "nauc_ndcg_at_20_std": -0.06976976094738162, + "nauc_ndcg_at_3_diff1": 0.45651566859484954, + "nauc_ndcg_at_3_max": -0.0917391957567451, + "nauc_ndcg_at_3_std": -0.0066655713004780326, + "nauc_ndcg_at_5_diff1": 0.42352618211285165, + "nauc_ndcg_at_5_max": -0.12048398138827213, + "nauc_ndcg_at_5_std": -0.09161503751671582, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.934640522875822, + "nauc_precision_at_10_max": 0.3384687208216678, + "nauc_precision_at_10_std": 0.3384687208216678, + "nauc_precision_at_1_diff1": 0.29789254971801715, + "nauc_precision_at_1_max": -0.21024042742653606, + "nauc_precision_at_1_std": -0.08892846542000597, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.7737034680241789, + "nauc_precision_at_3_max": 0.1622653515749282, + "nauc_precision_at_3_std": 0.19837734648425048, + "nauc_precision_at_5_diff1": 0.7373949579831942, + "nauc_precision_at_5_max": 0.11788048552754457, + "nauc_precision_at_5_std": -0.1762371615312792, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.9346405228758167, + "nauc_recall_at_10_max": 0.33846872082166374, + "nauc_recall_at_10_std": 0.33846872082166374, + "nauc_recall_at_1_diff1": 0.29789254971801715, + "nauc_recall_at_1_max": -0.21024042742653606, + "nauc_recall_at_1_std": -0.08892846542000597, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.7737034680241817, + "nauc_recall_at_3_max": 0.16226535157492855, + "nauc_recall_at_3_std": 0.19837734648424982, + "nauc_recall_at_5_diff1": 0.737394957983194, + "nauc_recall_at_5_max": 0.11788048552754304, + "nauc_recall_at_5_std": -0.17623716153127933, + "ndcg_at_1": 0.6, + "ndcg_at_10": 0.79725, + "ndcg_at_100": 0.80754, + "ndcg_at_1000": 0.80754, + "ndcg_at_20": 0.80754, + "ndcg_at_3": 0.7688, + "ndcg_at_5": 0.78516, + "precision_at_1": 0.6, + "precision_at_10": 0.096, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.29333, + "precision_at_5": 0.184, + "recall_at_1": 0.6, + "recall_at_10": 0.96, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.88, + "recall_at_5": 0.92 + } + ], + "test_512": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88, + "map_at_1": 0.88, + "map_at_10": 0.93667, + "map_at_100": 0.93667, + "map_at_1000": 0.93667, + "map_at_20": 0.93667, + "map_at_3": 0.93667, + "map_at_5": 0.93667, + "mrr_at_1": 0.88, + "mrr_at_10": 0.9366666666666668, + "mrr_at_100": 0.9366666666666668, + "mrr_at_1000": 0.9366666666666668, + "mrr_at_20": 0.9366666666666668, + "mrr_at_3": 0.9366666666666668, + "mrr_at_5": 0.9366666666666668, + "nauc_map_at_1000_diff1": 0.8137889507752522, + "nauc_map_at_1000_max": -0.04909930252395461, + "nauc_map_at_1000_std": -0.8598775653570124, + "nauc_map_at_100_diff1": 0.8137889507752522, + "nauc_map_at_100_max": -0.04909930252395461, + "nauc_map_at_100_std": -0.8598775653570124, + "nauc_map_at_10_diff1": 0.8137889507752522, + "nauc_map_at_10_max": -0.04909930252395461, + "nauc_map_at_10_std": -0.8598775653570124, + "nauc_map_at_1_diff1": 0.8032134902958963, + "nauc_map_at_1_max": -0.04215717467387855, + "nauc_map_at_1_std": -0.8433025771555845, + "nauc_map_at_20_diff1": 0.8137889507752522, + "nauc_map_at_20_max": -0.04909930252395461, + "nauc_map_at_20_std": -0.8598775653570124, + "nauc_map_at_3_diff1": 0.8137889507752522, + "nauc_map_at_3_max": -0.04909930252395461, + "nauc_map_at_3_std": -0.8598775653570124, + "nauc_map_at_5_diff1": 0.8137889507752522, + "nauc_map_at_5_max": -0.04909930252395461, + "nauc_map_at_5_std": -0.8598775653570124, + "nauc_mrr_at_1000_diff1": 0.8137889507752522, + "nauc_mrr_at_1000_max": -0.04909930252395461, + "nauc_mrr_at_1000_std": -0.8598775653570124, + "nauc_mrr_at_100_diff1": 0.8137889507752522, + "nauc_mrr_at_100_max": -0.04909930252395461, + "nauc_mrr_at_100_std": -0.8598775653570124, + "nauc_mrr_at_10_diff1": 0.8137889507752522, + "nauc_mrr_at_10_max": -0.04909930252395461, + "nauc_mrr_at_10_std": -0.8598775653570124, + "nauc_mrr_at_1_diff1": 0.8032134902958963, + "nauc_mrr_at_1_max": -0.04215717467387855, + "nauc_mrr_at_1_std": -0.8433025771555845, + "nauc_mrr_at_20_diff1": 0.8137889507752522, + "nauc_mrr_at_20_max": -0.04909930252395461, + "nauc_mrr_at_20_std": -0.8598775653570124, + "nauc_mrr_at_3_diff1": 0.8137889507752522, + "nauc_mrr_at_3_max": -0.04909930252395461, + "nauc_mrr_at_3_std": -0.8598775653570124, + "nauc_mrr_at_5_diff1": 0.8137889507752522, + "nauc_mrr_at_5_max": -0.04909930252395461, + "nauc_mrr_at_5_std": -0.8598775653570124, + "nauc_ndcg_at_1000_diff1": 0.8144298646155557, + "nauc_ndcg_at_1000_max": -0.04952002234310427, + "nauc_ndcg_at_1000_std": -0.8608820738001176, + "nauc_ndcg_at_100_diff1": 0.8144298646155557, + "nauc_ndcg_at_100_max": -0.04952002234310427, + "nauc_ndcg_at_100_std": -0.8608820738001176, + "nauc_ndcg_at_10_diff1": 0.8144298646155557, + "nauc_ndcg_at_10_max": -0.04952002234310427, + "nauc_ndcg_at_10_std": -0.8608820738001176, + "nauc_ndcg_at_1_diff1": 0.8032134902958963, + "nauc_ndcg_at_1_max": -0.04215717467387855, + "nauc_ndcg_at_1_std": -0.8433025771555845, + "nauc_ndcg_at_20_diff1": 0.8144298646155557, + "nauc_ndcg_at_20_max": -0.04952002234310427, + "nauc_ndcg_at_20_std": -0.8608820738001176, + "nauc_ndcg_at_3_diff1": 0.8144298646155557, + "nauc_ndcg_at_3_max": -0.04952002234310427, + "nauc_ndcg_at_3_std": -0.8608820738001176, + "nauc_ndcg_at_5_diff1": 0.8144298646155557, + "nauc_ndcg_at_5_max": -0.04952002234310427, + "nauc_ndcg_at_5_std": -0.8608820738001176, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 0.8032134902958963, + "nauc_precision_at_1_max": -0.04215717467387855, + "nauc_precision_at_1_std": -0.8433025771555845, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 0.8032134902958963, + "nauc_recall_at_1_max": -0.04215717467387855, + "nauc_recall_at_1_std": -0.8433025771555845, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.88, + "ndcg_at_10": 0.95309, + "ndcg_at_100": 0.95309, + "ndcg_at_1000": 0.95309, + "ndcg_at_20": 0.95309, + "ndcg_at_3": 0.95309, + "ndcg_at_5": 0.95309, + "precision_at_1": 0.88, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 0.88, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_8192": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.34, + "map_at_1": 0.34, + "map_at_10": 0.48857, + "map_at_100": 0.50104, + "map_at_1000": 0.50104, + "map_at_20": 0.4976, + "map_at_3": 0.45667, + "map_at_5": 0.47667, + "mrr_at_1": 0.34, + "mrr_at_10": 0.48857142857142855, + "mrr_at_100": 0.5010381249697254, + "mrr_at_1000": 0.5010381249697254, + "mrr_at_20": 0.49760382101558576, + "mrr_at_3": 0.45666666666666667, + "mrr_at_5": 0.4766666666666666, + "nauc_map_at_1000_diff1": 0.22993693252442793, + "nauc_map_at_1000_max": 0.04937348972339396, + "nauc_map_at_1000_std": 0.3273643969111535, + "nauc_map_at_100_diff1": 0.22993693252442793, + "nauc_map_at_100_max": 0.04937348972339396, + "nauc_map_at_100_std": 0.3273643969111535, + "nauc_map_at_10_diff1": 0.22386790165770787, + "nauc_map_at_10_max": 0.04811156717353681, + "nauc_map_at_10_std": 0.3272823788067511, + "nauc_map_at_1_diff1": 0.2699859561726596, + "nauc_map_at_1_max": 0.07331937783195083, + "nauc_map_at_1_std": 0.3041945997509207, + "nauc_map_at_20_diff1": 0.2301409290250162, + "nauc_map_at_20_max": 0.042434161871118384, + "nauc_map_at_20_std": 0.3280432084401891, + "nauc_map_at_3_diff1": 0.2035379908444709, + "nauc_map_at_3_max": 0.057555611787168226, + "nauc_map_at_3_std": 0.30673782521681137, + "nauc_map_at_5_diff1": 0.220136705464889, + "nauc_map_at_5_max": 0.049302440625401414, + "nauc_map_at_5_std": 0.3240071729937737, + "nauc_mrr_at_1000_diff1": 0.22993693252442793, + "nauc_mrr_at_1000_max": 0.04937348972339396, + "nauc_mrr_at_1000_std": 0.3273643969111535, + "nauc_mrr_at_100_diff1": 0.22993693252442793, + "nauc_mrr_at_100_max": 0.04937348972339396, + "nauc_mrr_at_100_std": 0.3273643969111535, + "nauc_mrr_at_10_diff1": 0.22386790165770787, + "nauc_mrr_at_10_max": 0.04811156717353681, + "nauc_mrr_at_10_std": 0.3272823788067511, + "nauc_mrr_at_1_diff1": 0.2699859561726596, + "nauc_mrr_at_1_max": 0.07331937783195083, + "nauc_mrr_at_1_std": 0.3041945997509207, + "nauc_mrr_at_20_diff1": 0.2301409290250162, + "nauc_mrr_at_20_max": 0.042434161871118384, + "nauc_mrr_at_20_std": 0.3280432084401891, + "nauc_mrr_at_3_diff1": 0.2035379908444709, + "nauc_mrr_at_3_max": 0.057555611787168226, + "nauc_mrr_at_3_std": 0.30673782521681137, + "nauc_mrr_at_5_diff1": 0.220136705464889, + "nauc_mrr_at_5_max": 0.049302440625401414, + "nauc_mrr_at_5_std": 0.3240071729937737, + "nauc_ndcg_at_1000_diff1": 0.2322397765295328, + "nauc_ndcg_at_1000_max": 0.043299590950082824, + "nauc_ndcg_at_1000_std": 0.33378571953815933, + "nauc_ndcg_at_100_diff1": 0.2322397765295328, + "nauc_ndcg_at_100_max": 0.043299590950082824, + "nauc_ndcg_at_100_std": 0.33378571953815933, + "nauc_ndcg_at_10_diff1": 0.222088332344727, + "nauc_ndcg_at_10_max": 0.031948592398418324, + "nauc_ndcg_at_10_std": 0.3425222283777138, + "nauc_ndcg_at_1_diff1": 0.2699859561726596, + "nauc_ndcg_at_1_max": 0.07331937783195083, + "nauc_ndcg_at_1_std": 0.3041945997509207, + "nauc_ndcg_at_20_diff1": 0.24125490407430547, + "nauc_ndcg_at_20_max": 0.003926754405734867, + "nauc_ndcg_at_20_std": 0.3433268867622521, + "nauc_ndcg_at_3_diff1": 0.18497474582088821, + "nauc_ndcg_at_3_max": 0.049135104363080495, + "nauc_ndcg_at_3_std": 0.30292047830707036, + "nauc_ndcg_at_5_diff1": 0.21371450443562903, + "nauc_ndcg_at_5_max": 0.033769125783782154, + "nauc_ndcg_at_5_std": 0.3337661148648367, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 0.22488535086960262, + "nauc_precision_at_10_max": -0.0395863978541152, + "nauc_precision_at_10_std": 0.4150298520377255, + "nauc_precision_at_1_diff1": 0.2699859561726596, + "nauc_precision_at_1_max": 0.07331937783195083, + "nauc_precision_at_1_std": 0.3041945997509207, + "nauc_precision_at_20_diff1": 0.388982259570495, + "nauc_precision_at_20_max": -0.43015873015872724, + "nauc_precision_at_20_std": 0.4857142857142858, + "nauc_precision_at_3_diff1": 0.12861383199762497, + "nauc_precision_at_3_max": 0.021964974769961166, + "nauc_precision_at_3_std": 0.28952211338676126, + "nauc_precision_at_5_diff1": 0.19670375233225101, + "nauc_precision_at_5_max": -0.02384078501831214, + "nauc_precision_at_5_std": 0.3703268606177874, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.22488535086960265, + "nauc_recall_at_10_max": -0.03958639785411462, + "nauc_recall_at_10_std": 0.4150298520377263, + "nauc_recall_at_1_diff1": 0.2699859561726596, + "nauc_recall_at_1_max": 0.07331937783195083, + "nauc_recall_at_1_std": 0.3041945997509207, + "nauc_recall_at_20_diff1": 0.3889822595704945, + "nauc_recall_at_20_max": -0.4301587301587318, + "nauc_recall_at_20_std": 0.4857142857142856, + "nauc_recall_at_3_diff1": 0.1286138319976255, + "nauc_recall_at_3_max": 0.02196497476996117, + "nauc_recall_at_3_std": 0.28952211338676154, + "nauc_recall_at_5_diff1": 0.19670375233225043, + "nauc_recall_at_5_max": -0.023840785018313184, + "nauc_recall_at_5_std": 0.3703268606177869, + "ndcg_at_1": 0.34, + "ndcg_at_10": 0.55515, + "ndcg_at_100": 0.60997, + "ndcg_at_1000": 0.60997, + "ndcg_at_20": 0.58969, + "ndcg_at_3": 0.49357, + "ndcg_at_5": 0.52802, + "precision_at_1": 0.34, + "precision_at_10": 0.076, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.045, + "precision_at_3": 0.2, + "precision_at_5": 0.136, + "recall_at_1": 0.34, + "recall_at_10": 0.76, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.9, + "recall_at_3": 0.6, + "recall_at_5": 0.68 + } + ] + }, + "task_name": "LEMBNeedleRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LEMBPasskeyRetrieval.json b/results/voyageai__voyage-3/no_revision_available/LEMBPasskeyRetrieval.json new file mode 100644 index 000000000..a33949b2c --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LEMBPasskeyRetrieval.json @@ -0,0 +1,1201 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 73.91097903251648, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test_1024": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_16384": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.98, + "map_at_1": 0.98, + "map_at_10": 0.99, + "map_at_100": 0.99, + "map_at_1000": 0.99, + "map_at_20": 0.99, + "map_at_3": 0.99, + "map_at_5": 0.99, + "mrr_at_1": 0.98, + "mrr_at_10": 0.99, + "mrr_at_100": 0.99, + "mrr_at_1000": 0.99, + "mrr_at_20": 0.99, + "mrr_at_3": 0.99, + "mrr_at_5": 0.99, + "nauc_map_at_1000_diff1": 1.0, + "nauc_map_at_1000_max": 0.7222222222222201, + "nauc_map_at_1000_std": -0.5634920634920657, + "nauc_map_at_100_diff1": 1.0, + "nauc_map_at_100_max": 0.7222222222222201, + "nauc_map_at_100_std": -0.5634920634920657, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_10_max": 0.7222222222222201, + "nauc_map_at_10_std": -0.5634920634920657, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_1_max": 0.7222222222222248, + "nauc_map_at_1_std": -0.5634920634920563, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_20_max": 0.7222222222222201, + "nauc_map_at_20_std": -0.5634920634920657, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_3_max": 0.7222222222222201, + "nauc_map_at_3_std": -0.5634920634920657, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_5_max": 0.7222222222222201, + "nauc_map_at_5_std": -0.5634920634920657, + "nauc_mrr_at_1000_diff1": 1.0, + "nauc_mrr_at_1000_max": 0.7222222222222201, + "nauc_mrr_at_1000_std": -0.5634920634920657, + "nauc_mrr_at_100_diff1": 1.0, + "nauc_mrr_at_100_max": 0.7222222222222201, + "nauc_mrr_at_100_std": -0.5634920634920657, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_10_max": 0.7222222222222201, + "nauc_mrr_at_10_std": -0.5634920634920657, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_1_max": 0.7222222222222248, + "nauc_mrr_at_1_std": -0.5634920634920563, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_20_max": 0.7222222222222201, + "nauc_mrr_at_20_std": -0.5634920634920657, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_3_max": 0.7222222222222201, + "nauc_mrr_at_3_std": -0.5634920634920657, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_5_max": 0.7222222222222201, + "nauc_mrr_at_5_std": -0.5634920634920657, + "nauc_ndcg_at_1000_diff1": 1.0, + "nauc_ndcg_at_1000_max": 0.7222222222222193, + "nauc_ndcg_at_1000_std": -0.5634920634920513, + "nauc_ndcg_at_100_diff1": 1.0, + "nauc_ndcg_at_100_max": 0.7222222222222193, + "nauc_ndcg_at_100_std": -0.5634920634920513, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.7222222222222193, + "nauc_ndcg_at_10_std": -0.5634920634920513, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_1_max": 0.7222222222222248, + "nauc_ndcg_at_1_std": -0.5634920634920563, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.7222222222222193, + "nauc_ndcg_at_20_std": -0.5634920634920513, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.7222222222222193, + "nauc_ndcg_at_3_std": -0.5634920634920513, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.7222222222222193, + "nauc_ndcg_at_5_std": -0.5634920634920513, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_1_max": 0.7222222222222248, + "nauc_precision_at_1_std": -0.5634920634920563, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_1_max": 0.7222222222222248, + "nauc_recall_at_1_std": -0.5634920634920563, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 0.98, + "ndcg_at_10": 0.99262, + "ndcg_at_100": 0.99262, + "ndcg_at_1000": 0.99262, + "ndcg_at_20": 0.99262, + "ndcg_at_3": 0.99262, + "ndcg_at_5": 0.99262, + "precision_at_1": 0.98, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 0.98, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_2048": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_256": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_32768": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.6, + "map_at_1": 0.6, + "map_at_10": 0.6, + "map_at_100": 0.60551, + "map_at_1000": 0.60551, + "map_at_20": 0.6, + "map_at_3": 0.6, + "map_at_5": 0.6, + "mrr_at_1": 0.6, + "mrr_at_10": 0.6, + "mrr_at_100": 0.6054363849991761, + "mrr_at_1000": 0.6054363849991761, + "mrr_at_20": 0.6, + "mrr_at_3": 0.6, + "mrr_at_5": 0.6, + "nauc_map_at_1000_diff1": 0.9995338971820894, + "nauc_map_at_1000_max": 0.9744609023583464, + "nauc_map_at_1000_std": 0.14286029882615642, + "nauc_map_at_100_diff1": 0.9995338971820894, + "nauc_map_at_100_max": 0.9744609023583464, + "nauc_map_at_100_std": 0.14286029882615642, + "nauc_map_at_10_diff1": 1.0, + "nauc_map_at_10_max": 0.9750667853962599, + "nauc_map_at_10_std": 0.14333630157316707, + "nauc_map_at_1_diff1": 1.0, + "nauc_map_at_1_max": 0.9750667853962599, + "nauc_map_at_1_std": 0.14333630157316707, + "nauc_map_at_20_diff1": 1.0, + "nauc_map_at_20_max": 0.9750667853962599, + "nauc_map_at_20_std": 0.14333630157316707, + "nauc_map_at_3_diff1": 1.0, + "nauc_map_at_3_max": 0.9750667853962599, + "nauc_map_at_3_std": 0.14333630157316707, + "nauc_map_at_5_diff1": 1.0, + "nauc_map_at_5_max": 0.9750667853962599, + "nauc_map_at_5_std": 0.14333630157316707, + "nauc_mrr_at_1000_diff1": 0.999428062514683, + "nauc_mrr_at_1000_max": 0.9746479246346655, + "nauc_mrr_at_1000_std": 0.14324179213595506, + "nauc_mrr_at_100_diff1": 0.999428062514683, + "nauc_mrr_at_100_max": 0.9746479246346655, + "nauc_mrr_at_100_std": 0.14324179213595506, + "nauc_mrr_at_10_diff1": 1.0, + "nauc_mrr_at_10_max": 0.9750667853962599, + "nauc_mrr_at_10_std": 0.14333630157316707, + "nauc_mrr_at_1_diff1": 1.0, + "nauc_mrr_at_1_max": 0.9750667853962599, + "nauc_mrr_at_1_std": 0.14333630157316707, + "nauc_mrr_at_20_diff1": 1.0, + "nauc_mrr_at_20_max": 0.9750667853962599, + "nauc_mrr_at_20_std": 0.14333630157316707, + "nauc_mrr_at_3_diff1": 1.0, + "nauc_mrr_at_3_max": 0.9750667853962599, + "nauc_mrr_at_3_std": 0.14333630157316707, + "nauc_mrr_at_5_diff1": 1.0, + "nauc_mrr_at_5_max": 0.9750667853962599, + "nauc_mrr_at_5_std": 0.14333630157316707, + "nauc_ndcg_at_1000_diff1": 0.9985694224965265, + "nauc_ndcg_at_1000_max": 0.9731828610111941, + "nauc_ndcg_at_1000_std": 0.14185787074998224, + "nauc_ndcg_at_100_diff1": 0.9985694224965265, + "nauc_ndcg_at_100_max": 0.9731828610111941, + "nauc_ndcg_at_100_std": 0.14185787074998224, + "nauc_ndcg_at_10_diff1": 1.0, + "nauc_ndcg_at_10_max": 0.9750667853962599, + "nauc_ndcg_at_10_std": 0.14333630157316707, + "nauc_ndcg_at_1_diff1": 1.0, + "nauc_ndcg_at_1_max": 0.9750667853962599, + "nauc_ndcg_at_1_std": 0.14333630157316707, + "nauc_ndcg_at_20_diff1": 1.0, + "nauc_ndcg_at_20_max": 0.9750667853962599, + "nauc_ndcg_at_20_std": 0.14333630157316707, + "nauc_ndcg_at_3_diff1": 1.0, + "nauc_ndcg_at_3_max": 0.9750667853962599, + "nauc_ndcg_at_3_std": 0.14333630157316707, + "nauc_ndcg_at_5_diff1": 1.0, + "nauc_ndcg_at_5_max": 0.9750667853962599, + "nauc_ndcg_at_5_std": 0.14333630157316707, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 0.9750667853962597, + "nauc_precision_at_10_std": 0.14333630157316699, + "nauc_precision_at_1_diff1": 1.0, + "nauc_precision_at_1_max": 0.9750667853962599, + "nauc_precision_at_1_std": 0.14333630157316707, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 0.9750667853962597, + "nauc_precision_at_20_std": 0.14333630157316699, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 0.9750667853962599, + "nauc_precision_at_3_std": 0.14333630157316707, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 0.9750667853962597, + "nauc_precision_at_5_std": 0.14333630157316699, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_10_max": 0.9750667853962599, + "nauc_recall_at_10_std": 0.14333630157316707, + "nauc_recall_at_1_diff1": 1.0, + "nauc_recall_at_1_max": 0.9750667853962599, + "nauc_recall_at_1_std": 0.14333630157316707, + "nauc_recall_at_20_diff1": 1.0, + "nauc_recall_at_20_max": 0.9750667853962599, + "nauc_recall_at_20_std": 0.14333630157316707, + "nauc_recall_at_3_diff1": 1.0, + "nauc_recall_at_3_max": 0.9750667853962599, + "nauc_recall_at_3_std": 0.14333630157316707, + "nauc_recall_at_5_diff1": 1.0, + "nauc_recall_at_5_max": 0.9750667853962599, + "nauc_recall_at_5_std": 0.14333630157316707, + "ndcg_at_1": 0.6, + "ndcg_at_10": 0.6, + "ndcg_at_100": 0.66437, + "ndcg_at_1000": 0.66437, + "ndcg_at_20": 0.6, + "ndcg_at_3": 0.6, + "ndcg_at_5": 0.6, + "precision_at_1": 0.6, + "precision_at_10": 0.06, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.03, + "precision_at_3": 0.2, + "precision_at_5": 0.12, + "recall_at_1": 0.6, + "recall_at_10": 0.6, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.6, + "recall_at_3": 0.6, + "recall_at_5": 0.6 + } + ], + "test_4096": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_512": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ], + "test_8192": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 1.0, + "map_at_1": 1.0, + "map_at_10": 1.0, + "map_at_100": 1.0, + "map_at_1000": 1.0, + "map_at_20": 1.0, + "map_at_3": 1.0, + "map_at_5": 1.0, + "mrr_at_1": 1.0, + "mrr_at_10": 1.0, + "mrr_at_100": 1.0, + "mrr_at_1000": 1.0, + "mrr_at_20": 1.0, + "mrr_at_3": 1.0, + "mrr_at_5": 1.0, + "nauc_map_at_1000_diff1": NaN, + "nauc_map_at_1000_max": NaN, + "nauc_map_at_1000_std": NaN, + "nauc_map_at_100_diff1": NaN, + "nauc_map_at_100_max": NaN, + "nauc_map_at_100_std": NaN, + "nauc_map_at_10_diff1": NaN, + "nauc_map_at_10_max": NaN, + "nauc_map_at_10_std": NaN, + "nauc_map_at_1_diff1": NaN, + "nauc_map_at_1_max": NaN, + "nauc_map_at_1_std": NaN, + "nauc_map_at_20_diff1": NaN, + "nauc_map_at_20_max": NaN, + "nauc_map_at_20_std": NaN, + "nauc_map_at_3_diff1": NaN, + "nauc_map_at_3_max": NaN, + "nauc_map_at_3_std": NaN, + "nauc_map_at_5_diff1": NaN, + "nauc_map_at_5_max": NaN, + "nauc_map_at_5_std": NaN, + "nauc_mrr_at_1000_diff1": NaN, + "nauc_mrr_at_1000_max": NaN, + "nauc_mrr_at_1000_std": NaN, + "nauc_mrr_at_100_diff1": NaN, + "nauc_mrr_at_100_max": NaN, + "nauc_mrr_at_100_std": NaN, + "nauc_mrr_at_10_diff1": NaN, + "nauc_mrr_at_10_max": NaN, + "nauc_mrr_at_10_std": NaN, + "nauc_mrr_at_1_diff1": NaN, + "nauc_mrr_at_1_max": NaN, + "nauc_mrr_at_1_std": NaN, + "nauc_mrr_at_20_diff1": NaN, + "nauc_mrr_at_20_max": NaN, + "nauc_mrr_at_20_std": NaN, + "nauc_mrr_at_3_diff1": NaN, + "nauc_mrr_at_3_max": NaN, + "nauc_mrr_at_3_std": NaN, + "nauc_mrr_at_5_diff1": NaN, + "nauc_mrr_at_5_max": NaN, + "nauc_mrr_at_5_std": NaN, + "nauc_ndcg_at_1000_diff1": NaN, + "nauc_ndcg_at_1000_max": NaN, + "nauc_ndcg_at_1000_std": NaN, + "nauc_ndcg_at_100_diff1": NaN, + "nauc_ndcg_at_100_max": NaN, + "nauc_ndcg_at_100_std": NaN, + "nauc_ndcg_at_10_diff1": NaN, + "nauc_ndcg_at_10_max": NaN, + "nauc_ndcg_at_10_std": NaN, + "nauc_ndcg_at_1_diff1": NaN, + "nauc_ndcg_at_1_max": NaN, + "nauc_ndcg_at_1_std": NaN, + "nauc_ndcg_at_20_diff1": NaN, + "nauc_ndcg_at_20_max": NaN, + "nauc_ndcg_at_20_std": NaN, + "nauc_ndcg_at_3_diff1": NaN, + "nauc_ndcg_at_3_max": NaN, + "nauc_ndcg_at_3_std": NaN, + "nauc_ndcg_at_5_diff1": NaN, + "nauc_ndcg_at_5_max": NaN, + "nauc_ndcg_at_5_std": NaN, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 1.0, + "nauc_precision_at_100_max": 1.0, + "nauc_precision_at_100_std": 1.0, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": 1.0, + "nauc_precision_at_1_diff1": NaN, + "nauc_precision_at_1_max": NaN, + "nauc_precision_at_1_std": NaN, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 1.0, + "nauc_precision_at_3_max": 1.0, + "nauc_precision_at_3_std": 1.0, + "nauc_precision_at_5_diff1": 1.0, + "nauc_precision_at_5_max": 1.0, + "nauc_precision_at_5_std": 1.0, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": NaN, + "nauc_recall_at_10_max": NaN, + "nauc_recall_at_10_std": NaN, + "nauc_recall_at_1_diff1": NaN, + "nauc_recall_at_1_max": NaN, + "nauc_recall_at_1_std": NaN, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": NaN, + "nauc_recall_at_3_max": NaN, + "nauc_recall_at_3_std": NaN, + "nauc_recall_at_5_diff1": NaN, + "nauc_recall_at_5_max": NaN, + "nauc_recall_at_5_std": NaN, + "ndcg_at_1": 1.0, + "ndcg_at_10": 1.0, + "ndcg_at_100": 1.0, + "ndcg_at_1000": 1.0, + "ndcg_at_20": 1.0, + "ndcg_at_3": 1.0, + "ndcg_at_5": 1.0, + "precision_at_1": 1.0, + "precision_at_10": 0.1, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.33333, + "precision_at_5": 0.2, + "recall_at_1": 1.0, + "recall_at_10": 1.0, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 1.0, + "recall_at_5": 1.0 + } + ] + }, + "task_name": "LEMBPasskeyRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LEMBQMSumRetrieval.json b/results/voyageai__voyage-3/no_revision_available/LEMBQMSumRetrieval.json new file mode 100644 index 000000000..8fd1939bd --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LEMBQMSumRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 42.843952655792236, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.51047, + "map_at_1": 0.34774, + "map_at_10": 0.45369, + "map_at_100": 0.46524, + "map_at_1000": 0.46541, + "map_at_20": 0.46011, + "map_at_3": 0.42338, + "map_at_5": 0.44037, + "mrr_at_1": 0.3477406679764244, + "mrr_at_10": 0.4536878514776358, + "mrr_at_100": 0.4652398119444461, + "mrr_at_1000": 0.4654072222595389, + "mrr_at_20": 0.46011100381534786, + "mrr_at_3": 0.4233791748526523, + "mrr_at_5": 0.4403732809430258, + "nauc_map_at_1000_diff1": 0.5762888612587302, + "nauc_map_at_1000_max": -0.045285260944256375, + "nauc_map_at_1000_std": -0.38273569269438457, + "nauc_map_at_100_diff1": 0.5762697170481254, + "nauc_map_at_100_max": -0.045300723897373656, + "nauc_map_at_100_std": -0.38278758006688113, + "nauc_map_at_10_diff1": 0.5746573028901543, + "nauc_map_at_10_max": -0.04824516408005318, + "nauc_map_at_10_std": -0.3861101871056484, + "nauc_map_at_1_diff1": 0.610042324839289, + "nauc_map_at_1_max": -0.018768783812946607, + "nauc_map_at_1_std": -0.337429480508177, + "nauc_map_at_20_diff1": 0.5748366577577841, + "nauc_map_at_20_max": -0.04778838042631749, + "nauc_map_at_20_std": -0.385808269698879, + "nauc_map_at_3_diff1": 0.5802696673374208, + "nauc_map_at_3_max": -0.0427652781524902, + "nauc_map_at_3_std": -0.375942591314725, + "nauc_map_at_5_diff1": 0.576187877646306, + "nauc_map_at_5_max": -0.04700705613306303, + "nauc_map_at_5_std": -0.382682633411973, + "nauc_mrr_at_1000_diff1": 0.5762888612587302, + "nauc_mrr_at_1000_max": -0.045285260944256375, + "nauc_mrr_at_1000_std": -0.38273569269438457, + "nauc_mrr_at_100_diff1": 0.5762697170481254, + "nauc_mrr_at_100_max": -0.045300723897373656, + "nauc_mrr_at_100_std": -0.38278758006688113, + "nauc_mrr_at_10_diff1": 0.5746573028901543, + "nauc_mrr_at_10_max": -0.04824516408005318, + "nauc_mrr_at_10_std": -0.3861101871056484, + "nauc_mrr_at_1_diff1": 0.610042324839289, + "nauc_mrr_at_1_max": -0.018768783812946607, + "nauc_mrr_at_1_std": -0.337429480508177, + "nauc_mrr_at_20_diff1": 0.5748366577577841, + "nauc_mrr_at_20_max": -0.04778838042631749, + "nauc_mrr_at_20_std": -0.385808269698879, + "nauc_mrr_at_3_diff1": 0.5802696673374208, + "nauc_mrr_at_3_max": -0.0427652781524902, + "nauc_mrr_at_3_std": -0.375942591314725, + "nauc_mrr_at_5_diff1": 0.576187877646306, + "nauc_mrr_at_5_max": -0.04700705613306303, + "nauc_mrr_at_5_std": -0.382682633411973, + "nauc_ndcg_at_1000_diff1": 0.5700852056090664, + "nauc_ndcg_at_1000_max": -0.0492511572749851, + "nauc_ndcg_at_1000_std": -0.3914582390750449, + "nauc_ndcg_at_100_diff1": 0.5698057572884213, + "nauc_ndcg_at_100_max": -0.04949110013095625, + "nauc_ndcg_at_100_std": -0.3923582676580985, + "nauc_ndcg_at_10_diff1": 0.5586618305118287, + "nauc_ndcg_at_10_max": -0.062402974725196715, + "nauc_ndcg_at_10_std": -0.4102356619806399, + "nauc_ndcg_at_1_diff1": 0.610042324839289, + "nauc_ndcg_at_1_max": -0.018768783812946607, + "nauc_ndcg_at_1_std": -0.337429480508177, + "nauc_ndcg_at_20_diff1": 0.558886695295201, + "nauc_ndcg_at_20_max": -0.06255103035338412, + "nauc_ndcg_at_20_std": -0.4117425917515555, + "nauc_ndcg_at_3_diff1": 0.5715009239022747, + "nauc_ndcg_at_3_max": -0.050475742838213065, + "nauc_ndcg_at_3_std": -0.3881985331802053, + "nauc_ndcg_at_5_diff1": 0.5641375800583722, + "nauc_ndcg_at_5_max": -0.058045414348809596, + "nauc_ndcg_at_5_std": -0.4005779731384728, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.5429460082230262, + "nauc_precision_at_100_max": -0.07256634787434481, + "nauc_precision_at_100_std": -0.4808614035057742, + "nauc_precision_at_10_diff1": 0.4950132847089664, + "nauc_precision_at_10_max": -0.11856757637296661, + "nauc_precision_at_10_std": -0.5060756129674012, + "nauc_precision_at_1_diff1": 0.610042324839289, + "nauc_precision_at_1_max": -0.018768783812946607, + "nauc_precision_at_1_std": -0.337429480508177, + "nauc_precision_at_20_diff1": 0.4768790633073118, + "nauc_precision_at_20_max": -0.14137040847858648, + "nauc_precision_at_20_std": -0.5505999873185767, + "nauc_precision_at_3_diff1": 0.5459654811297576, + "nauc_precision_at_3_max": -0.07315283784236935, + "nauc_precision_at_3_std": -0.4242063458678077, + "nauc_precision_at_5_diff1": 0.5257985861431067, + "nauc_precision_at_5_max": -0.09348222956895277, + "nauc_precision_at_5_std": -0.4582763530557747, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.5429460082230225, + "nauc_recall_at_100_max": -0.07256634787435523, + "nauc_recall_at_100_std": -0.4808614035057827, + "nauc_recall_at_10_diff1": 0.49501328470896583, + "nauc_recall_at_10_max": -0.11856757637296693, + "nauc_recall_at_10_std": -0.5060756129674013, + "nauc_recall_at_1_diff1": 0.610042324839289, + "nauc_recall_at_1_max": -0.018768783812946607, + "nauc_recall_at_1_std": -0.337429480508177, + "nauc_recall_at_20_diff1": 0.47687906330731256, + "nauc_recall_at_20_max": -0.14137040847858623, + "nauc_recall_at_20_std": -0.550599987318577, + "nauc_recall_at_3_diff1": 0.5459654811297581, + "nauc_recall_at_3_max": -0.07315283784236896, + "nauc_recall_at_3_std": -0.42420634586780753, + "nauc_recall_at_5_diff1": 0.5257985861431062, + "nauc_recall_at_5_max": -0.09348222956895262, + "nauc_recall_at_5_std": -0.45827635305577463, + "ndcg_at_1": 0.34774, + "ndcg_at_10": 0.51047, + "ndcg_at_100": 0.57062, + "ndcg_at_1000": 0.57332, + "ndcg_at_20": 0.53373, + "ndcg_at_3": 0.44755, + "ndcg_at_5": 0.47824, + "precision_at_1": 0.34774, + "precision_at_10": 0.06916, + "precision_at_100": 0.00982, + "precision_at_1000": 0.001, + "precision_at_20": 0.03916, + "precision_at_3": 0.17245, + "precision_at_5": 0.1184, + "recall_at_1": 0.34774, + "recall_at_10": 0.69155, + "recall_at_100": 0.98166, + "recall_at_1000": 1.0, + "recall_at_20": 0.78324, + "recall_at_3": 0.51735, + "recall_at_5": 0.59201 + } + ] + }, + "task_name": "LEMBQMSumRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LEMBSummScreenFDRetrieval.json b/results/voyageai__voyage-3/no_revision_available/LEMBSummScreenFDRetrieval.json new file mode 100644 index 000000000..38a989401 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LEMBSummScreenFDRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 42.815211057662964, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "validation": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.9782, + "map_at_1": 0.95536, + "map_at_10": 0.97199, + "map_at_100": 0.97223, + "map_at_1000": 0.97223, + "map_at_20": 0.97223, + "map_at_3": 0.96974, + "map_at_5": 0.97123, + "mrr_at_1": 0.9553571428571429, + "mrr_at_10": 0.9719860166288737, + "mrr_at_100": 0.9722340325018896, + "mrr_at_1000": 0.9722340325018896, + "mrr_at_20": 0.9722340325018896, + "mrr_at_3": 0.9697420634920636, + "mrr_at_5": 0.9712301587301588, + "nauc_map_at_1000_diff1": 0.9567150124535275, + "nauc_map_at_1000_max": 0.8597404285135241, + "nauc_map_at_1000_std": -0.19886143296619563, + "nauc_map_at_100_diff1": 0.9567150124535275, + "nauc_map_at_100_max": 0.8597404285135241, + "nauc_map_at_100_std": -0.19886143296619563, + "nauc_map_at_10_diff1": 0.9570982269301241, + "nauc_map_at_10_max": 0.8609821869457448, + "nauc_map_at_10_std": -0.19862390183335013, + "nauc_map_at_1_diff1": 0.9552988097427586, + "nauc_map_at_1_max": 0.8704947100084144, + "nauc_map_at_1_std": -0.15734704907790908, + "nauc_map_at_20_diff1": 0.9567150124535275, + "nauc_map_at_20_max": 0.8597404285135241, + "nauc_map_at_20_std": -0.19886143296619563, + "nauc_map_at_3_diff1": 0.9584406241095034, + "nauc_map_at_3_max": 0.8592378888825987, + "nauc_map_at_3_std": -0.20647635410816784, + "nauc_map_at_5_diff1": 0.9562910012186158, + "nauc_map_at_5_max": 0.8519570900316946, + "nauc_map_at_5_std": -0.21588716962052754, + "nauc_mrr_at_1000_diff1": 0.9567150124535275, + "nauc_mrr_at_1000_max": 0.8597404285135241, + "nauc_mrr_at_1000_std": -0.19886143296619563, + "nauc_mrr_at_100_diff1": 0.9567150124535275, + "nauc_mrr_at_100_max": 0.8597404285135241, + "nauc_mrr_at_100_std": -0.19886143296619563, + "nauc_mrr_at_10_diff1": 0.9570982269301241, + "nauc_mrr_at_10_max": 0.8609821869457448, + "nauc_mrr_at_10_std": -0.19862390183335013, + "nauc_mrr_at_1_diff1": 0.9552988097427586, + "nauc_mrr_at_1_max": 0.8704947100084144, + "nauc_mrr_at_1_std": -0.15734704907790908, + "nauc_mrr_at_20_diff1": 0.9567150124535275, + "nauc_mrr_at_20_max": 0.8597404285135241, + "nauc_mrr_at_20_std": -0.19886143296619563, + "nauc_mrr_at_3_diff1": 0.9584406241095034, + "nauc_mrr_at_3_max": 0.8592378888825987, + "nauc_mrr_at_3_std": -0.20647635410816784, + "nauc_mrr_at_5_diff1": 0.9562910012186158, + "nauc_mrr_at_5_max": 0.8519570900316946, + "nauc_mrr_at_5_std": -0.21588716962052754, + "nauc_ndcg_at_1000_diff1": 0.956976651525155, + "nauc_ndcg_at_1000_max": 0.8583285285782201, + "nauc_ndcg_at_1000_std": -0.2058831507928328, + "nauc_ndcg_at_100_diff1": 0.956976651525155, + "nauc_ndcg_at_100_max": 0.8583285285782201, + "nauc_ndcg_at_100_std": -0.2058831507928328, + "nauc_ndcg_at_10_diff1": 0.958563695472954, + "nauc_ndcg_at_10_max": 0.8635545014340127, + "nauc_ndcg_at_10_std": -0.2046344404806955, + "nauc_ndcg_at_1_diff1": 0.9552988097427586, + "nauc_ndcg_at_1_max": 0.8704947100084144, + "nauc_ndcg_at_1_std": -0.15734704907790908, + "nauc_ndcg_at_20_diff1": 0.956976651525155, + "nauc_ndcg_at_20_max": 0.8583285285782201, + "nauc_ndcg_at_20_std": -0.2058831507928328, + "nauc_ndcg_at_3_diff1": 0.9606433859816896, + "nauc_ndcg_at_3_max": 0.8529590843917891, + "nauc_ndcg_at_3_std": -0.23803000343094205, + "nauc_ndcg_at_5_diff1": 0.9563847486032114, + "nauc_ndcg_at_5_max": 0.8370483167865411, + "nauc_ndcg_at_5_std": -0.2611316451761894, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 1.0, + "nauc_precision_at_10_max": 1.0, + "nauc_precision_at_10_std": -0.1720317259612042, + "nauc_precision_at_1_diff1": 0.9552988097427586, + "nauc_precision_at_1_max": 0.8704947100084144, + "nauc_precision_at_1_std": -0.15734704907790908, + "nauc_precision_at_20_diff1": NaN, + "nauc_precision_at_20_max": NaN, + "nauc_precision_at_20_std": NaN, + "nauc_precision_at_3_diff1": 0.9738215816127721, + "nauc_precision_at_3_max": 0.8160832847639696, + "nauc_precision_at_3_std": -0.42556988667709317, + "nauc_precision_at_5_diff1": 0.956369302687954, + "nauc_precision_at_5_max": 0.6934721412732934, + "nauc_precision_at_5_std": -0.6929271972157439, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 1.0, + "nauc_recall_at_10_max": 1.0, + "nauc_recall_at_10_std": -0.17203172596128666, + "nauc_recall_at_1_diff1": 0.9552988097427586, + "nauc_recall_at_1_max": 0.8704947100084144, + "nauc_recall_at_1_std": -0.15734704907790908, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.9738215816127851, + "nauc_recall_at_3_max": 0.8160832847639907, + "nauc_recall_at_3_std": -0.4255698866770906, + "nauc_recall_at_5_diff1": 0.9563693026879464, + "nauc_recall_at_5_max": 0.6934721412733068, + "nauc_recall_at_5_std": -0.6929271972157647, + "ndcg_at_1": 0.95536, + "ndcg_at_10": 0.9782, + "ndcg_at_100": 0.979, + "ndcg_at_1000": 0.979, + "ndcg_at_20": 0.979, + "ndcg_at_3": 0.97375, + "ndcg_at_5": 0.97631, + "precision_at_1": 0.95536, + "precision_at_10": 0.0997, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.32837, + "precision_at_5": 0.19821, + "recall_at_1": 0.95536, + "recall_at_10": 0.99702, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.98512, + "recall_at_5": 0.99107 + } + ] + }, + "task_name": "LEMBSummScreenFDRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LEMBWikimQARetrieval.json b/results/voyageai__voyage-3/no_revision_available/LEMBWikimQARetrieval.json new file mode 100644 index 000000000..2f2f63098 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LEMBWikimQARetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "6e346642246bfb4928c560ee08640dc84d074e8c", + "evaluation_time": 44.97714567184448, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.88903, + "map_at_1": 0.82667, + "map_at_10": 0.86956, + "map_at_100": 0.87159, + "map_at_1000": 0.87161, + "map_at_20": 0.87103, + "map_at_3": 0.86111, + "map_at_5": 0.86494, + "mrr_at_1": 0.8266666666666667, + "mrr_at_10": 0.8695634920634921, + "mrr_at_100": 0.8715937797611214, + "mrr_at_1000": 0.8716133876042587, + "mrr_at_20": 0.8710295858109327, + "mrr_at_3": 0.8611111111111113, + "mrr_at_5": 0.8649444444444445, + "nauc_map_at_1000_diff1": 0.8644055031903924, + "nauc_map_at_1000_max": 0.7263386281343154, + "nauc_map_at_1000_std": -0.08632723615465511, + "nauc_map_at_100_diff1": 0.8643828965871552, + "nauc_map_at_100_max": 0.7263112515887445, + "nauc_map_at_100_std": -0.0865901579235437, + "nauc_map_at_10_diff1": 0.8627343470353461, + "nauc_map_at_10_max": 0.7265447536442814, + "nauc_map_at_10_std": -0.0875571393397368, + "nauc_map_at_1_diff1": 0.8762354595909688, + "nauc_map_at_1_max": 0.6922755264958569, + "nauc_map_at_1_std": -0.10221052231430071, + "nauc_map_at_20_diff1": 0.8645783408158932, + "nauc_map_at_20_max": 0.7271221085637845, + "nauc_map_at_20_std": -0.08372196730381243, + "nauc_map_at_3_diff1": 0.8621729017251412, + "nauc_map_at_3_max": 0.7303741035084327, + "nauc_map_at_3_std": -0.08334367125411807, + "nauc_map_at_5_diff1": 0.8638250862767056, + "nauc_map_at_5_max": 0.7296596704742679, + "nauc_map_at_5_std": -0.08113462598394534, + "nauc_mrr_at_1000_diff1": 0.8644055031903924, + "nauc_mrr_at_1000_max": 0.7263386281343154, + "nauc_mrr_at_1000_std": -0.08632723615465511, + "nauc_mrr_at_100_diff1": 0.8643828965871552, + "nauc_mrr_at_100_max": 0.7263112515887445, + "nauc_mrr_at_100_std": -0.0865901579235437, + "nauc_mrr_at_10_diff1": 0.8627343470353461, + "nauc_mrr_at_10_max": 0.7265447536442814, + "nauc_mrr_at_10_std": -0.0875571393397368, + "nauc_mrr_at_1_diff1": 0.8762354595909688, + "nauc_mrr_at_1_max": 0.6922755264958569, + "nauc_mrr_at_1_std": -0.10221052231430071, + "nauc_mrr_at_20_diff1": 0.8645783408158932, + "nauc_mrr_at_20_max": 0.7271221085637845, + "nauc_mrr_at_20_std": -0.08372196730381243, + "nauc_mrr_at_3_diff1": 0.8621729017251412, + "nauc_mrr_at_3_max": 0.7303741035084327, + "nauc_mrr_at_3_std": -0.08334367125411807, + "nauc_mrr_at_5_diff1": 0.8638250862767056, + "nauc_mrr_at_5_max": 0.7296596704742679, + "nauc_mrr_at_5_std": -0.08113462598394534, + "nauc_ndcg_at_1000_diff1": 0.863279189757633, + "nauc_ndcg_at_1000_max": 0.7299977430685162, + "nauc_ndcg_at_1000_std": -0.08879441543594777, + "nauc_ndcg_at_100_diff1": 0.8626274034433843, + "nauc_ndcg_at_100_max": 0.7291852210184726, + "nauc_ndcg_at_100_std": -0.09642401982727346, + "nauc_ndcg_at_10_diff1": 0.8567590241155768, + "nauc_ndcg_at_10_max": 0.7331074316031829, + "nauc_ndcg_at_10_std": -0.09502906154996858, + "nauc_ndcg_at_1_diff1": 0.8762354595909688, + "nauc_ndcg_at_1_max": 0.6922755264958569, + "nauc_ndcg_at_1_std": -0.10221052231430071, + "nauc_ndcg_at_20_diff1": 0.8649642324632615, + "nauc_ndcg_at_20_max": 0.7356739785799682, + "nauc_ndcg_at_20_std": -0.07774895984295685, + "nauc_ndcg_at_3_diff1": 0.8576396656268319, + "nauc_ndcg_at_3_max": 0.7414891245181167, + "nauc_ndcg_at_3_std": -0.08073151933571958, + "nauc_ndcg_at_5_diff1": 0.8601772418692885, + "nauc_ndcg_at_5_max": 0.7404514463149264, + "nauc_ndcg_at_5_std": -0.07618323663117958, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7222222222222392, + "nauc_precision_at_100_max": 0.554154995331414, + "nauc_precision_at_100_std": -1.739962651727316, + "nauc_precision_at_10_diff1": 0.8115779645191415, + "nauc_precision_at_10_max": 0.7696856520385927, + "nauc_precision_at_10_std": -0.17214441332088215, + "nauc_precision_at_1_diff1": 0.8762354595909688, + "nauc_precision_at_1_max": 0.6922755264958569, + "nauc_precision_at_1_std": -0.10221052231430071, + "nauc_precision_at_20_diff1": 0.897875816993463, + "nauc_precision_at_20_max": 0.8374766573295949, + "nauc_precision_at_20_std": 0.025676937441645196, + "nauc_precision_at_3_diff1": 0.8406629318394017, + "nauc_precision_at_3_max": 0.7825241207594142, + "nauc_precision_at_3_std": -0.07270463741051987, + "nauc_precision_at_5_diff1": 0.8444631185807657, + "nauc_precision_at_5_max": 0.7862745098039221, + "nauc_precision_at_5_std": -0.055387488328664664, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.722222222222207, + "nauc_recall_at_100_max": 0.554154995331476, + "nauc_recall_at_100_std": -1.739962651727313, + "nauc_recall_at_10_diff1": 0.8115779645191386, + "nauc_recall_at_10_max": 0.7696856520385936, + "nauc_recall_at_10_std": -0.1721444133208865, + "nauc_recall_at_1_diff1": 0.8762354595909688, + "nauc_recall_at_1_max": 0.6922755264958569, + "nauc_recall_at_1_std": -0.10221052231430071, + "nauc_recall_at_20_diff1": 0.8978758169934609, + "nauc_recall_at_20_max": 0.8374766573295934, + "nauc_recall_at_20_std": 0.02567693744163807, + "nauc_recall_at_3_diff1": 0.8406629318394019, + "nauc_recall_at_3_max": 0.7825241207594147, + "nauc_recall_at_3_std": -0.07270463741052104, + "nauc_recall_at_5_diff1": 0.8444631185807647, + "nauc_recall_at_5_max": 0.7862745098039229, + "nauc_recall_at_5_std": -0.05538748832866262, + "ndcg_at_1": 0.82667, + "ndcg_at_10": 0.88903, + "ndcg_at_100": 0.89898, + "ndcg_at_1000": 0.89943, + "ndcg_at_20": 0.89474, + "ndcg_at_3": 0.87119, + "ndcg_at_5": 0.87807, + "precision_at_1": 0.82667, + "precision_at_10": 0.095, + "precision_at_100": 0.00997, + "precision_at_1000": 0.001, + "precision_at_20": 0.04867, + "precision_at_3": 0.3, + "precision_at_5": 0.18333, + "recall_at_1": 0.82667, + "recall_at_10": 0.95, + "recall_at_100": 0.99667, + "recall_at_1000": 1.0, + "recall_at_20": 0.97333, + "recall_at_3": 0.9, + "recall_at_5": 0.91667 + } + ] + }, + "task_name": "LEMBWikimQARetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LeCaRDv2.json b/results/voyageai__voyage-3/no_revision_available/LeCaRDv2.json new file mode 100644 index 000000000..7089fb665 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LeCaRDv2.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b78e18688c3d012a33dc3676597c1d1b2243ce1c", + "evaluation_time": 233.74614691734314, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "zho-Hans" + ], + "main_score": 0.69659, + "map_at_1": 0.03632, + "map_at_10": 0.23592, + "map_at_100": 0.54115, + "map_at_1000": 0.55595, + "map_at_20": 0.36096, + "map_at_3": 0.09462, + "map_at_5": 0.14321, + "mrr_at_1": 0.7987421383647799, + "mrr_at_10": 0.8727463312368974, + "mrr_at_100": 0.8736499288108611, + "mrr_at_1000": 0.8736499288108611, + "mrr_at_20": 0.8735355777531139, + "mrr_at_3": 0.8647798742138365, + "mrr_at_5": 0.8716981132075473, + "nauc_map_at_1000_diff1": -0.04355973369297658, + "nauc_map_at_1000_max": 0.39268200405393494, + "nauc_map_at_1000_std": 0.27824759160507523, + "nauc_map_at_100_diff1": -0.03871435009471337, + "nauc_map_at_100_max": 0.3874444250117557, + "nauc_map_at_100_std": 0.27446126256184367, + "nauc_map_at_10_diff1": 0.09558684118577966, + "nauc_map_at_10_max": 0.3064503646634594, + "nauc_map_at_10_std": 0.012409414336684736, + "nauc_map_at_1_diff1": 0.4031299582489357, + "nauc_map_at_1_max": 0.22256896735163073, + "nauc_map_at_1_std": -0.20792011771839983, + "nauc_map_at_20_diff1": -0.003451667511835026, + "nauc_map_at_20_max": 0.3502181693987153, + "nauc_map_at_20_std": 0.14711774821456544, + "nauc_map_at_3_diff1": 0.19415316067645558, + "nauc_map_at_3_max": 0.24442820310314017, + "nauc_map_at_3_std": -0.09967998312896827, + "nauc_map_at_5_diff1": 0.18118231681413846, + "nauc_map_at_5_max": 0.26782710556628875, + "nauc_map_at_5_std": -0.08763747448770423, + "nauc_mrr_at_1000_diff1": 0.5215479905419076, + "nauc_mrr_at_1000_max": 0.5972533052697838, + "nauc_mrr_at_1000_std": 0.12998787257202654, + "nauc_mrr_at_100_diff1": 0.5215479905419076, + "nauc_mrr_at_100_max": 0.5972533052697838, + "nauc_mrr_at_100_std": 0.12998787257202654, + "nauc_mrr_at_10_diff1": 0.5199596011858433, + "nauc_mrr_at_10_max": 0.5975702156473518, + "nauc_mrr_at_10_std": 0.13380368790244432, + "nauc_mrr_at_1_diff1": 0.5225868385780573, + "nauc_mrr_at_1_max": 0.5651656383701618, + "nauc_mrr_at_1_std": 0.07574641698862013, + "nauc_mrr_at_20_diff1": 0.5213905332523245, + "nauc_mrr_at_20_max": 0.5976374681572615, + "nauc_mrr_at_20_std": 0.13081774001820703, + "nauc_mrr_at_3_diff1": 0.5288704726999219, + "nauc_mrr_at_3_max": 0.6026217836052097, + "nauc_mrr_at_3_std": 0.12548095446556007, + "nauc_mrr_at_5_diff1": 0.5240935646841315, + "nauc_mrr_at_5_max": 0.598640899643816, + "nauc_mrr_at_5_std": 0.13368214862775687, + "nauc_ndcg_at_1000_diff1": 0.009883112648997001, + "nauc_ndcg_at_1000_max": 0.42125059632803247, + "nauc_ndcg_at_1000_std": 0.29443287479898306, + "nauc_ndcg_at_100_diff1": -0.014086029113585516, + "nauc_ndcg_at_100_max": 0.3710177850864883, + "nauc_ndcg_at_100_std": 0.2985134068964106, + "nauc_ndcg_at_10_diff1": 0.03309894794380563, + "nauc_ndcg_at_10_max": 0.43284881100023304, + "nauc_ndcg_at_10_std": 0.22418378163318997, + "nauc_ndcg_at_1_diff1": 0.5225868385780573, + "nauc_ndcg_at_1_max": 0.5651656383701618, + "nauc_ndcg_at_1_std": 0.07574641698862013, + "nauc_ndcg_at_20_diff1": -0.03806110259667491, + "nauc_ndcg_at_20_max": 0.40107844715967517, + "nauc_ndcg_at_20_std": 0.2496236631479006, + "nauc_ndcg_at_3_diff1": 0.15732529202742188, + "nauc_ndcg_at_3_max": 0.529285629918103, + "nauc_ndcg_at_3_std": 0.20786897287715517, + "nauc_ndcg_at_5_diff1": 0.12097663451354276, + "nauc_ndcg_at_5_max": 0.4702952036986209, + "nauc_ndcg_at_5_std": 0.18213976944660362, + "nauc_precision_at_1000_diff1": -0.15359785654646838, + "nauc_precision_at_1000_max": 0.3215069039130171, + "nauc_precision_at_1000_std": 0.46259458753708294, + "nauc_precision_at_100_diff1": -0.15213743857030937, + "nauc_precision_at_100_max": 0.2959340206085989, + "nauc_precision_at_100_std": 0.44999490979841156, + "nauc_precision_at_10_diff1": -0.04171804185734592, + "nauc_precision_at_10_max": 0.37782127396268744, + "nauc_precision_at_10_std": 0.2501733548413824, + "nauc_precision_at_1_diff1": 0.5225868385780573, + "nauc_precision_at_1_max": 0.5651656383701618, + "nauc_precision_at_1_std": 0.07574641698862013, + "nauc_precision_at_20_diff1": -0.10633898319614597, + "nauc_precision_at_20_max": 0.3651911189124957, + "nauc_precision_at_20_std": 0.313415706204312, + "nauc_precision_at_3_diff1": 0.06202396866525049, + "nauc_precision_at_3_max": 0.5096399376953821, + "nauc_precision_at_3_std": 0.23735598450732132, + "nauc_precision_at_5_diff1": 0.04371547210958, + "nauc_precision_at_5_max": 0.41992985714351905, + "nauc_precision_at_5_std": 0.18688151776549802, + "nauc_recall_at_1000_diff1": -1.2997474349994074, + "nauc_recall_at_1000_max": -0.8174626549165738, + "nauc_recall_at_1000_std": 0.6606301894554799, + "nauc_recall_at_100_diff1": -0.1322227360590368, + "nauc_recall_at_100_max": 0.12930823527683574, + "nauc_recall_at_100_std": 0.2939632708678423, + "nauc_recall_at_10_diff1": 0.05298932630356948, + "nauc_recall_at_10_max": 0.12374917685376739, + "nauc_recall_at_10_std": -0.05355596297930751, + "nauc_recall_at_1_diff1": 0.4031299582489357, + "nauc_recall_at_1_max": 0.22256896735163073, + "nauc_recall_at_1_std": -0.20792011771839983, + "nauc_recall_at_20_diff1": -0.040998535563412625, + "nauc_recall_at_20_max": 0.19514463117759778, + "nauc_recall_at_20_std": 0.07231994369717828, + "nauc_recall_at_3_diff1": 0.12951414045183812, + "nauc_recall_at_3_max": 0.1502681302672191, + "nauc_recall_at_3_std": -0.11551400056314229, + "nauc_recall_at_5_diff1": 0.12594750581940056, + "nauc_recall_at_5_max": 0.11246660725422533, + "nauc_recall_at_5_std": -0.14899550545021492, + "ndcg_at_1": 0.79874, + "ndcg_at_10": 0.69659, + "ndcg_at_100": 0.7708, + "ndcg_at_1000": 0.81824, + "ndcg_at_20": 0.63328, + "ndcg_at_3": 0.77313, + "ndcg_at_5": 0.75305, + "precision_at_1": 0.79874, + "precision_at_10": 0.65597, + "precision_at_100": 0.21591, + "precision_at_1000": 0.02445, + "precision_at_20": 0.55126, + "precision_at_3": 0.7673, + "precision_at_5": 0.73836, + "recall_at_1": 0.03632, + "recall_at_10": 0.27772, + "recall_at_100": 0.87865, + "recall_at_1000": 0.9979, + "recall_at_20": 0.45736, + "recall_at_3": 0.10246, + "recall_at_5": 0.16035 + } + ] + }, + "task_name": "LeCaRDv2" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LegalBenchConsumerContractsQA.json b/results/voyageai__voyage-3/no_revision_available/LegalBenchConsumerContractsQA.json new file mode 100644 index 000000000..0d79951c4 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LegalBenchConsumerContractsQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "b23590301ec94e8087e2850b21d43d4956b1cca9", + "evaluation_time": 2.9258034229278564, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.86752, + "map_at_1": 0.75505, + "map_at_10": 0.83487, + "map_at_100": 0.83655, + "map_at_1000": 0.83655, + "map_at_20": 0.83609, + "map_at_3": 0.82492, + "map_at_5": 0.83136, + "mrr_at_1": 0.76010101010101, + "mrr_at_10": 0.8373907728074396, + "mrr_at_100": 0.8390777509810806, + "mrr_at_1000": 0.8390777509810806, + "mrr_at_20": 0.8386181273242677, + "mrr_at_3": 0.8274410774410773, + "mrr_at_5": 0.8338804713804713, + "nauc_map_at_1000_diff1": 0.6809764407211033, + "nauc_map_at_1000_max": 0.5229515531094573, + "nauc_map_at_1000_std": 0.11774009451729944, + "nauc_map_at_100_diff1": 0.6809764407211033, + "nauc_map_at_100_max": 0.5229515531094573, + "nauc_map_at_100_std": 0.11774009451729944, + "nauc_map_at_10_diff1": 0.6783737877932466, + "nauc_map_at_10_max": 0.5251888377653143, + "nauc_map_at_10_std": 0.12363555535558138, + "nauc_map_at_1_diff1": 0.7057100672478187, + "nauc_map_at_1_max": 0.4986232185678854, + "nauc_map_at_1_std": 0.09352888466822053, + "nauc_map_at_20_diff1": 0.6809300009432084, + "nauc_map_at_20_max": 0.5234131340351649, + "nauc_map_at_20_std": 0.11902952733851684, + "nauc_map_at_3_diff1": 0.6789320781335637, + "nauc_map_at_3_max": 0.5309974569457276, + "nauc_map_at_3_std": 0.11973657930018917, + "nauc_map_at_5_diff1": 0.6757367494916784, + "nauc_map_at_5_max": 0.5262972185790333, + "nauc_map_at_5_std": 0.12112677217396181, + "nauc_mrr_at_1000_diff1": 0.6723072132932955, + "nauc_mrr_at_1000_max": 0.5323263699623016, + "nauc_mrr_at_1000_std": 0.11508662886664053, + "nauc_mrr_at_100_diff1": 0.6723072132932955, + "nauc_mrr_at_100_max": 0.5323263699623016, + "nauc_mrr_at_100_std": 0.11508662886664053, + "nauc_mrr_at_10_diff1": 0.6697705371656362, + "nauc_mrr_at_10_max": 0.5344852777096838, + "nauc_mrr_at_10_std": 0.12108379296525104, + "nauc_mrr_at_1_diff1": 0.6942331009173465, + "nauc_mrr_at_1_max": 0.5110751729852864, + "nauc_mrr_at_1_std": 0.08947883999206664, + "nauc_mrr_at_20_diff1": 0.6722870356280795, + "nauc_mrr_at_20_max": 0.5327646030714077, + "nauc_mrr_at_20_std": 0.11640010955239208, + "nauc_mrr_at_3_diff1": 0.6708712881336353, + "nauc_mrr_at_3_max": 0.539780958800534, + "nauc_mrr_at_3_std": 0.11729929173969261, + "nauc_mrr_at_5_diff1": 0.6672991967224808, + "nauc_mrr_at_5_max": 0.5353938888070213, + "nauc_mrr_at_5_std": 0.11860360804285398, + "nauc_ndcg_at_1000_diff1": 0.6762920762634785, + "nauc_ndcg_at_1000_max": 0.5265776731527956, + "nauc_ndcg_at_1000_std": 0.12604202533717845, + "nauc_ndcg_at_100_diff1": 0.6762920762634785, + "nauc_ndcg_at_100_max": 0.5265776731527956, + "nauc_ndcg_at_100_std": 0.12604202533717845, + "nauc_ndcg_at_10_diff1": 0.6625916121387707, + "nauc_ndcg_at_10_max": 0.537060887161065, + "nauc_ndcg_at_10_std": 0.15742599788554348, + "nauc_ndcg_at_1_diff1": 0.7057100672478187, + "nauc_ndcg_at_1_max": 0.4986232185678854, + "nauc_ndcg_at_1_std": 0.09352888466822053, + "nauc_ndcg_at_20_diff1": 0.6751054248054832, + "nauc_ndcg_at_20_max": 0.5306879156383669, + "nauc_ndcg_at_20_std": 0.13682471648106193, + "nauc_ndcg_at_3_diff1": 0.6643840861844418, + "nauc_ndcg_at_3_max": 0.5470210025860494, + "nauc_ndcg_at_3_std": 0.14203362915191647, + "nauc_ndcg_at_5_diff1": 0.6566321486876386, + "nauc_ndcg_at_5_max": 0.538496882539172, + "nauc_ndcg_at_5_std": 0.14804246853843456, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": 0.45415394395919984, + "nauc_precision_at_10_max": 0.6921648101472868, + "nauc_precision_at_10_std": 0.643423930829526, + "nauc_precision_at_1_diff1": 0.7057100672478187, + "nauc_precision_at_1_max": 0.4986232185678854, + "nauc_precision_at_1_std": 0.09352888466822053, + "nauc_precision_at_20_diff1": 0.6159207134315247, + "nauc_precision_at_20_max": 0.7005558903981208, + "nauc_precision_at_20_std": 0.5685745528481583, + "nauc_precision_at_3_diff1": 0.5869116908600922, + "nauc_precision_at_3_max": 0.6316389295422209, + "nauc_precision_at_3_std": 0.2644401712131634, + "nauc_precision_at_5_diff1": 0.5123582292436728, + "nauc_precision_at_5_max": 0.6259701649437158, + "nauc_precision_at_5_std": 0.35747574877214483, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": 0.4541539439592013, + "nauc_recall_at_10_max": 0.6921648101472879, + "nauc_recall_at_10_std": 0.6434239308295294, + "nauc_recall_at_1_diff1": 0.7057100672478187, + "nauc_recall_at_1_max": 0.4986232185678854, + "nauc_recall_at_1_std": 0.09352888466822053, + "nauc_recall_at_20_diff1": 0.6159207134315192, + "nauc_recall_at_20_max": 0.7005558903981158, + "nauc_recall_at_20_std": 0.5685745528481603, + "nauc_recall_at_3_diff1": 0.5869116908600912, + "nauc_recall_at_3_max": 0.6316389295422221, + "nauc_recall_at_3_std": 0.2644401712131628, + "nauc_recall_at_5_diff1": 0.5123582292436732, + "nauc_recall_at_5_max": 0.6259701649437124, + "nauc_recall_at_5_std": 0.3574757487721478, + "ndcg_at_1": 0.75505, + "ndcg_at_10": 0.86752, + "ndcg_at_100": 0.87493, + "ndcg_at_1000": 0.87493, + "ndcg_at_20": 0.87199, + "ndcg_at_3": 0.84722, + "ndcg_at_5": 0.85874, + "precision_at_1": 0.75505, + "precision_at_10": 0.09672, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.04924, + "precision_at_3": 0.30387, + "precision_at_5": 0.18788, + "recall_at_1": 0.75505, + "recall_at_10": 0.96717, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 0.98485, + "recall_at_3": 0.91162, + "recall_at_5": 0.93939 + } + ] + }, + "task_name": "LegalBenchConsumerContractsQA" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LegalBenchCorporateLobbying.json b/results/voyageai__voyage-3/no_revision_available/LegalBenchCorporateLobbying.json new file mode 100644 index 000000000..9c4f744a2 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LegalBenchCorporateLobbying.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "f69691c650464e62546d7f2a4536f8f87c891e38", + "evaluation_time": 3.10664701461792, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94543, + "map_at_1": 0.88235, + "map_at_10": 0.92975, + "map_at_100": 0.92981, + "map_at_1000": 0.92985, + "map_at_20": 0.92975, + "map_at_3": 0.92794, + "map_at_5": 0.92926, + "mrr_at_1": 0.8823529411764706, + "mrr_at_10": 0.9297549019607841, + "mrr_at_100": 0.9298103958564556, + "mrr_at_1000": 0.9298461129307122, + "mrr_at_20": 0.9297549019607841, + "mrr_at_3": 0.927941176470588, + "mrr_at_5": 0.9292647058823528, + "nauc_map_at_1000_diff1": 0.9045117020717914, + "nauc_map_at_1000_max": 0.5090290292609364, + "nauc_map_at_1000_std": -0.28110646093634895, + "nauc_map_at_100_diff1": 0.9043899828823607, + "nauc_map_at_100_max": 0.509036662162998, + "nauc_map_at_100_std": -0.28149529532957934, + "nauc_map_at_10_diff1": 0.9043617836656529, + "nauc_map_at_10_max": 0.50943099997814, + "nauc_map_at_10_std": -0.2804660085867059, + "nauc_map_at_1_diff1": 0.9076785714285707, + "nauc_map_at_1_max": 0.43780952380952387, + "nauc_map_at_1_std": -0.32379761904761895, + "nauc_map_at_20_diff1": 0.9043617836656529, + "nauc_map_at_20_max": 0.50943099997814, + "nauc_map_at_20_std": -0.2804660085867059, + "nauc_map_at_3_diff1": 0.9048200003873044, + "nauc_map_at_3_max": 0.5210758018810075, + "nauc_map_at_3_std": -0.2885190134071797, + "nauc_map_at_5_diff1": 0.90411464407569, + "nauc_map_at_5_max": 0.5119661003793436, + "nauc_map_at_5_std": -0.2907479873415804, + "nauc_mrr_at_1000_diff1": 0.9045117020717914, + "nauc_mrr_at_1000_max": 0.5090290292609364, + "nauc_mrr_at_1000_std": -0.28110646093634895, + "nauc_mrr_at_100_diff1": 0.9043899828823607, + "nauc_mrr_at_100_max": 0.509036662162998, + "nauc_mrr_at_100_std": -0.28149529532957934, + "nauc_mrr_at_10_diff1": 0.9043617836656529, + "nauc_mrr_at_10_max": 0.50943099997814, + "nauc_mrr_at_10_std": -0.2804660085867059, + "nauc_mrr_at_1_diff1": 0.9076785714285707, + "nauc_mrr_at_1_max": 0.43780952380952387, + "nauc_mrr_at_1_std": -0.32379761904761895, + "nauc_mrr_at_20_diff1": 0.9043617836656529, + "nauc_mrr_at_20_max": 0.50943099997814, + "nauc_mrr_at_20_std": -0.2804660085867059, + "nauc_mrr_at_3_diff1": 0.9048200003873044, + "nauc_mrr_at_3_max": 0.5210758018810075, + "nauc_mrr_at_3_std": -0.2885190134071797, + "nauc_mrr_at_5_diff1": 0.90411464407569, + "nauc_mrr_at_5_max": 0.5119661003793436, + "nauc_mrr_at_5_std": -0.2907479873415804, + "nauc_ndcg_at_1000_diff1": 0.9028810602007047, + "nauc_ndcg_at_1000_max": 0.5180104706472388, + "nauc_ndcg_at_1000_std": -0.2735535414038619, + "nauc_ndcg_at_100_diff1": 0.9006943240829675, + "nauc_ndcg_at_100_max": 0.5199134079454012, + "nauc_ndcg_at_100_std": -0.2770018699165497, + "nauc_ndcg_at_10_diff1": 0.9003954480014508, + "nauc_ndcg_at_10_max": 0.5244811069263025, + "nauc_ndcg_at_10_std": -0.2648520614520688, + "nauc_ndcg_at_1_diff1": 0.9076785714285707, + "nauc_ndcg_at_1_max": 0.43780952380952387, + "nauc_ndcg_at_1_std": -0.32379761904761895, + "nauc_ndcg_at_20_diff1": 0.9003954480014508, + "nauc_ndcg_at_20_max": 0.5244811069263025, + "nauc_ndcg_at_20_std": -0.2648520614520688, + "nauc_ndcg_at_3_diff1": 0.9014129498638479, + "nauc_ndcg_at_3_max": 0.5508001510715275, + "nauc_ndcg_at_3_std": -0.2879515897509883, + "nauc_ndcg_at_5_diff1": 0.8998002108049272, + "nauc_ndcg_at_5_max": 0.5310773367783351, + "nauc_ndcg_at_5_std": -0.2930718127193484, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.7770774976657122, + "nauc_precision_at_100_max": 0.6790382819794465, + "nauc_precision_at_100_std": -0.3699813258637037, + "nauc_precision_at_10_diff1": 0.8078120136943786, + "nauc_precision_at_10_max": 0.7860255213196584, + "nauc_precision_at_10_std": 0.08667911609089299, + "nauc_precision_at_1_diff1": 0.9076785714285707, + "nauc_precision_at_1_max": 0.43780952380952387, + "nauc_precision_at_1_std": -0.32379761904761895, + "nauc_precision_at_20_diff1": 0.8078120136943786, + "nauc_precision_at_20_max": 0.7860255213196584, + "nauc_precision_at_20_std": 0.08667911609089299, + "nauc_precision_at_3_diff1": 0.8603330220977249, + "nauc_precision_at_3_max": 0.8712262682850961, + "nauc_precision_at_3_std": -0.3037659508247634, + "nauc_precision_at_5_diff1": 0.8231792717086898, + "nauc_precision_at_5_max": 0.8068394024276497, + "nauc_precision_at_5_std": -0.3699813258636605, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.7770774976657118, + "nauc_recall_at_100_max": 0.6790382819794377, + "nauc_recall_at_100_std": -0.36998132586367716, + "nauc_recall_at_10_diff1": 0.8078120136943808, + "nauc_recall_at_10_max": 0.7860255213196531, + "nauc_recall_at_10_std": 0.08667911609087513, + "nauc_recall_at_1_diff1": 0.9076785714285707, + "nauc_recall_at_1_max": 0.43780952380952387, + "nauc_recall_at_1_std": -0.32379761904761895, + "nauc_recall_at_20_diff1": 0.8078120136943808, + "nauc_recall_at_20_max": 0.7860255213196531, + "nauc_recall_at_20_std": 0.08667911609087513, + "nauc_recall_at_3_diff1": 0.8603330220977297, + "nauc_recall_at_3_max": 0.8712262682850933, + "nauc_recall_at_3_std": -0.3037659508247648, + "nauc_recall_at_5_diff1": 0.8231792717086823, + "nauc_recall_at_5_max": 0.8068394024276364, + "nauc_recall_at_5_std": -0.36998132586368127, + "ndcg_at_1": 0.88235, + "ndcg_at_10": 0.94543, + "ndcg_at_100": 0.94594, + "ndcg_at_1000": 0.94673, + "ndcg_at_20": 0.94543, + "ndcg_at_3": 0.94198, + "ndcg_at_5": 0.94438, + "precision_at_1": 0.88235, + "precision_at_10": 0.09912, + "precision_at_100": 0.00994, + "precision_at_1000": 0.001, + "precision_at_20": 0.04956, + "precision_at_3": 0.32745, + "precision_at_5": 0.19765, + "recall_at_1": 0.88235, + "recall_at_10": 0.99118, + "recall_at_100": 0.99412, + "recall_at_1000": 1.0, + "recall_at_20": 0.99118, + "recall_at_3": 0.98235, + "recall_at_5": 0.98824 + } + ] + }, + "task_name": "LegalBenchCorporateLobbying" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LegalQuAD.json b/results/voyageai__voyage-3/no_revision_available/LegalQuAD.json new file mode 100644 index 000000000..6a4ae88ac --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LegalQuAD.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e", + "evaluation_time": 17.151992321014404, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "deu-Latn" + ], + "main_score": 0.65029, + "map_at_1": 0.51, + "map_at_10": 0.60336, + "map_at_100": 0.6107, + "map_at_1000": 0.6109, + "map_at_20": 0.60835, + "map_at_3": 0.5825, + "map_at_5": 0.592, + "mrr_at_1": 0.51, + "mrr_at_10": 0.6033630952380953, + "mrr_at_100": 0.6107021272359894, + "mrr_at_1000": 0.6108978647251927, + "mrr_at_20": 0.6083541384011586, + "mrr_at_3": 0.5824999999999999, + "mrr_at_5": 0.592, + "nauc_map_at_1000_diff1": 0.7512606064769293, + "nauc_map_at_1000_max": 0.635350724802579, + "nauc_map_at_1000_std": 0.424522090608248, + "nauc_map_at_100_diff1": 0.7511994168287109, + "nauc_map_at_100_max": 0.6353377249930009, + "nauc_map_at_100_std": 0.42477201167394923, + "nauc_map_at_10_diff1": 0.748434586804131, + "nauc_map_at_10_max": 0.6358502263138017, + "nauc_map_at_10_std": 0.42870448632461433, + "nauc_map_at_1_diff1": 0.7662385741017462, + "nauc_map_at_1_max": 0.6163455972429766, + "nauc_map_at_1_std": 0.37368576928772385, + "nauc_map_at_20_diff1": 0.750051280479846, + "nauc_map_at_20_max": 0.6352778732848824, + "nauc_map_at_20_std": 0.42525423389924594, + "nauc_map_at_3_diff1": 0.7637723897302315, + "nauc_map_at_3_max": 0.6356555875812617, + "nauc_map_at_3_std": 0.4048152026314525, + "nauc_map_at_5_diff1": 0.7539559966007509, + "nauc_map_at_5_max": 0.6291582790397726, + "nauc_map_at_5_std": 0.411292558417059, + "nauc_mrr_at_1000_diff1": 0.7512606064769293, + "nauc_mrr_at_1000_max": 0.635350724802579, + "nauc_mrr_at_1000_std": 0.424522090608248, + "nauc_mrr_at_100_diff1": 0.7511994168287109, + "nauc_mrr_at_100_max": 0.6353377249930009, + "nauc_mrr_at_100_std": 0.42477201167394923, + "nauc_mrr_at_10_diff1": 0.748434586804131, + "nauc_mrr_at_10_max": 0.6358502263138017, + "nauc_mrr_at_10_std": 0.42870448632461433, + "nauc_mrr_at_1_diff1": 0.7662385741017462, + "nauc_mrr_at_1_max": 0.6163455972429766, + "nauc_mrr_at_1_std": 0.37368576928772385, + "nauc_mrr_at_20_diff1": 0.750051280479846, + "nauc_mrr_at_20_max": 0.6352778732848824, + "nauc_mrr_at_20_std": 0.42525423389924594, + "nauc_mrr_at_3_diff1": 0.7637723897302315, + "nauc_mrr_at_3_max": 0.6356555875812617, + "nauc_mrr_at_3_std": 0.4048152026314525, + "nauc_mrr_at_5_diff1": 0.7539559966007509, + "nauc_mrr_at_5_max": 0.6291582790397726, + "nauc_mrr_at_5_std": 0.411292558417059, + "nauc_ndcg_at_1000_diff1": 0.7461780182508032, + "nauc_ndcg_at_1000_max": 0.6385519373397759, + "nauc_ndcg_at_1000_std": 0.43856625745633937, + "nauc_ndcg_at_100_diff1": 0.7450356336781456, + "nauc_ndcg_at_100_max": 0.6386510180652119, + "nauc_ndcg_at_100_std": 0.4449871671848916, + "nauc_ndcg_at_10_diff1": 0.7294991015958298, + "nauc_ndcg_at_10_max": 0.6435049526711574, + "nauc_ndcg_at_10_std": 0.4667943051988635, + "nauc_ndcg_at_1_diff1": 0.7662385741017462, + "nauc_ndcg_at_1_max": 0.6163455972429766, + "nauc_ndcg_at_1_std": 0.37368576928772385, + "nauc_ndcg_at_20_diff1": 0.7357148860263228, + "nauc_ndcg_at_20_max": 0.6413452050241558, + "nauc_ndcg_at_20_std": 0.4541211346048438, + "nauc_ndcg_at_3_diff1": 0.7640362804129429, + "nauc_ndcg_at_3_max": 0.6389264417224326, + "nauc_ndcg_at_3_std": 0.41047319367912644, + "nauc_ndcg_at_5_diff1": 0.7466070020465793, + "nauc_ndcg_at_5_max": 0.6268101206794476, + "nauc_ndcg_at_5_std": 0.422080205838854, + "nauc_precision_at_1000_diff1": 1.0, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": 0.6841736694677922, + "nauc_precision_at_100_max": 0.6514939309057045, + "nauc_precision_at_100_std": 0.8187052598817316, + "nauc_precision_at_10_diff1": 0.6253121878121878, + "nauc_precision_at_10_max": 0.6839660339660331, + "nauc_precision_at_10_std": 0.6685189810189806, + "nauc_precision_at_1_diff1": 0.7662385741017462, + "nauc_precision_at_1_max": 0.6163455972429766, + "nauc_precision_at_1_std": 0.37368576928772385, + "nauc_precision_at_20_diff1": 0.6281752203214099, + "nauc_precision_at_20_max": 0.6851440420647263, + "nauc_precision_at_20_std": 0.6605013700659108, + "nauc_precision_at_3_diff1": 0.7653078498755733, + "nauc_precision_at_3_max": 0.648440012865873, + "nauc_precision_at_3_std": 0.42721470772460224, + "nauc_precision_at_5_diff1": 0.7188536589885662, + "nauc_precision_at_5_max": 0.6150139910679541, + "nauc_precision_at_5_std": 0.45958341826807236, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.6841736694677863, + "nauc_recall_at_100_max": 0.6514939309056984, + "nauc_recall_at_100_std": 0.8187052598817317, + "nauc_recall_at_10_diff1": 0.6253121878121874, + "nauc_recall_at_10_max": 0.6839660339660336, + "nauc_recall_at_10_std": 0.668518981018981, + "nauc_recall_at_1_diff1": 0.7662385741017462, + "nauc_recall_at_1_max": 0.6163455972429766, + "nauc_recall_at_1_std": 0.37368576928772385, + "nauc_recall_at_20_diff1": 0.6281752203214107, + "nauc_recall_at_20_max": 0.6851440420647268, + "nauc_recall_at_20_std": 0.6605013700659117, + "nauc_recall_at_3_diff1": 0.7653078498755733, + "nauc_recall_at_3_max": 0.648440012865873, + "nauc_recall_at_3_std": 0.42721470772460296, + "nauc_recall_at_5_diff1": 0.7188536589885661, + "nauc_recall_at_5_max": 0.6150139910679543, + "nauc_recall_at_5_std": 0.45958341826807264, + "ndcg_at_1": 0.51, + "ndcg_at_10": 0.65029, + "ndcg_at_100": 0.68633, + "ndcg_at_1000": 0.69045, + "ndcg_at_20": 0.66816, + "ndcg_at_3": 0.60506, + "ndcg_at_5": 0.62185, + "precision_at_1": 0.51, + "precision_at_10": 0.08, + "precision_at_100": 0.0097, + "precision_at_1000": 0.001, + "precision_at_20": 0.0435, + "precision_at_3": 0.22333, + "precision_at_5": 0.142, + "recall_at_1": 0.51, + "recall_at_10": 0.8, + "recall_at_100": 0.97, + "recall_at_1000": 1.0, + "recall_at_20": 0.87, + "recall_at_3": 0.67, + "recall_at_5": 0.71 + } + ] + }, + "task_name": "LegalQuAD" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/LegalSummarization.json b/results/voyageai__voyage-3/no_revision_available/LegalSummarization.json new file mode 100644 index 000000000..20f23f467 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/LegalSummarization.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "3bb1a05c66872889662af04c5691c14489cebd72", + "evaluation_time": 3.3200454711914062, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.69226, + "map_at_1": 0.50858, + "map_at_10": 0.63114, + "map_at_100": 0.64423, + "map_at_1000": 0.6445, + "map_at_20": 0.63906, + "map_at_3": 0.59265, + "map_at_5": 0.61294, + "mrr_at_1": 0.573943661971831, + "mrr_at_10": 0.6751592890677395, + "mrr_at_100": 0.6796068769267017, + "mrr_at_1000": 0.6797098980689807, + "mrr_at_20": 0.6786528671827692, + "mrr_at_3": 0.6473004694835679, + "mrr_at_5": 0.6656103286384975, + "nauc_map_at_1000_diff1": 0.7434364086666728, + "nauc_map_at_1000_max": 0.5978921882853642, + "nauc_map_at_1000_std": 0.013610498466275295, + "nauc_map_at_100_diff1": 0.7434515607233665, + "nauc_map_at_100_max": 0.597934031683507, + "nauc_map_at_100_std": 0.013476550182018208, + "nauc_map_at_10_diff1": 0.745560252852326, + "nauc_map_at_10_max": 0.5971795362234235, + "nauc_map_at_10_std": 0.000896576493465759, + "nauc_map_at_1_diff1": 0.8050069354935895, + "nauc_map_at_1_max": 0.5843175549790467, + "nauc_map_at_1_std": 0.0053998685161305415, + "nauc_map_at_20_diff1": 0.7427258675924447, + "nauc_map_at_20_max": 0.5975124804213073, + "nauc_map_at_20_std": 0.009733371677738358, + "nauc_map_at_3_diff1": 0.7464316247749531, + "nauc_map_at_3_max": 0.5777598598687017, + "nauc_map_at_3_std": -0.0165741165908828, + "nauc_map_at_5_diff1": 0.7459305927360906, + "nauc_map_at_5_max": 0.5880693987742083, + "nauc_map_at_5_std": -0.01193995735106696, + "nauc_mrr_at_1000_diff1": 0.7373787299372702, + "nauc_mrr_at_1000_max": 0.6101274780565479, + "nauc_mrr_at_1000_std": 0.052309867208183355, + "nauc_mrr_at_100_diff1": 0.7372273335979416, + "nauc_mrr_at_100_max": 0.6100505000337473, + "nauc_mrr_at_100_std": 0.05230275802387543, + "nauc_mrr_at_10_diff1": 0.7358682125394681, + "nauc_mrr_at_10_max": 0.6103459860976325, + "nauc_mrr_at_10_std": 0.0520701057819074, + "nauc_mrr_at_1_diff1": 0.7860969695089025, + "nauc_mrr_at_1_max": 0.604248738996377, + "nauc_mrr_at_1_std": 0.03455972007191286, + "nauc_mrr_at_20_diff1": 0.7367261764725448, + "nauc_mrr_at_20_max": 0.6108289205274756, + "nauc_mrr_at_20_std": 0.05432037197174845, + "nauc_mrr_at_3_diff1": 0.7389240952429608, + "nauc_mrr_at_3_max": 0.6102064805525286, + "nauc_mrr_at_3_std": 0.040954007128386725, + "nauc_mrr_at_5_diff1": 0.7377510088155805, + "nauc_mrr_at_5_max": 0.6135554283098597, + "nauc_mrr_at_5_std": 0.049184755672908476, + "nauc_ndcg_at_1000_diff1": 0.7275817364129734, + "nauc_ndcg_at_1000_max": 0.6038916227453268, + "nauc_ndcg_at_1000_std": 0.041597517212805295, + "nauc_ndcg_at_100_diff1": 0.7245055625037289, + "nauc_ndcg_at_100_max": 0.6032168174187389, + "nauc_ndcg_at_100_std": 0.04243061175515225, + "nauc_ndcg_at_10_diff1": 0.722522601969394, + "nauc_ndcg_at_10_max": 0.6066988616875734, + "nauc_ndcg_at_10_std": 0.021235473591254838, + "nauc_ndcg_at_1_diff1": 0.7860969695089025, + "nauc_ndcg_at_1_max": 0.604248738996377, + "nauc_ndcg_at_1_std": 0.03455972007191286, + "nauc_ndcg_at_20_diff1": 0.7152471448481124, + "nauc_ndcg_at_20_max": 0.6079803694538438, + "nauc_ndcg_at_20_std": 0.04699925431234948, + "nauc_ndcg_at_3_diff1": 0.7222525754329179, + "nauc_ndcg_at_3_max": 0.5961361579794409, + "nauc_ndcg_at_3_std": 0.008024118931235355, + "nauc_ndcg_at_5_diff1": 0.723588111804428, + "nauc_ndcg_at_5_max": 0.6071565051252165, + "nauc_ndcg_at_5_std": 0.013186612345468546, + "nauc_precision_at_1000_diff1": -0.3355944508763231, + "nauc_precision_at_1000_max": -0.15807998900375583, + "nauc_precision_at_1000_std": 0.1948226679749564, + "nauc_precision_at_100_diff1": -0.31156657736278076, + "nauc_precision_at_100_max": -0.1312265168719777, + "nauc_precision_at_100_std": 0.20109148387680423, + "nauc_precision_at_10_diff1": -0.09592139105364214, + "nauc_precision_at_10_max": 0.10643246648226797, + "nauc_precision_at_10_std": 0.153761400220905, + "nauc_precision_at_1_diff1": 0.7860969695089025, + "nauc_precision_at_1_max": 0.604248738996377, + "nauc_precision_at_1_std": 0.03455972007191286, + "nauc_precision_at_20_diff1": -0.25121111510768795, + "nauc_precision_at_20_max": -0.023511403413756727, + "nauc_precision_at_20_std": 0.22068768384781842, + "nauc_precision_at_3_diff1": 0.2948229060515225, + "nauc_precision_at_3_max": 0.3348255242884497, + "nauc_precision_at_3_std": 0.029101361383545844, + "nauc_precision_at_5_diff1": 0.10316542396874775, + "nauc_precision_at_5_max": 0.24505614553885463, + "nauc_precision_at_5_std": 0.09715207755996752, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.4561429921546896, + "nauc_recall_at_100_max": 0.5067597997678432, + "nauc_recall_at_100_std": 0.1262249737505333, + "nauc_recall_at_10_diff1": 0.5942064025135579, + "nauc_recall_at_10_max": 0.5974289650589174, + "nauc_recall_at_10_std": 0.04189718031251686, + "nauc_recall_at_1_diff1": 0.8050069354935895, + "nauc_recall_at_1_max": 0.5843175549790467, + "nauc_recall_at_1_std": 0.0053998685161305415, + "nauc_recall_at_20_diff1": 0.4825074720395449, + "nauc_recall_at_20_max": 0.6035595484011376, + "nauc_recall_at_20_std": 0.20969010594716264, + "nauc_recall_at_3_diff1": 0.6687254568059432, + "nauc_recall_at_3_max": 0.5770788277623664, + "nauc_recall_at_3_std": -0.014534125767499281, + "nauc_recall_at_5_diff1": 0.6414601994385234, + "nauc_recall_at_5_max": 0.6012227054184694, + "nauc_recall_at_5_std": -0.0042920671573146146, + "ndcg_at_1": 0.57394, + "ndcg_at_10": 0.69226, + "ndcg_at_100": 0.73258, + "ndcg_at_1000": 0.73633, + "ndcg_at_20": 0.71327, + "ndcg_at_3": 0.63526, + "ndcg_at_5": 0.6626, + "precision_at_1": 0.57394, + "precision_at_10": 0.11373, + "precision_at_100": 0.01504, + "precision_at_1000": 0.00155, + "precision_at_20": 0.06461, + "precision_at_3": 0.27465, + "precision_at_5": 0.19085, + "recall_at_1": 0.50858, + "recall_at_10": 0.8262, + "recall_at_100": 0.9754, + "recall_at_1000": 1.0, + "recall_at_20": 0.89395, + "recall_at_3": 0.66707, + "recall_at_5": 0.73782 + } + ] + }, + "task_name": "LegalSummarization" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/MIRACLRetrieval.json b/results/voyageai__voyage-3/no_revision_available/MIRACLRetrieval.json new file mode 100644 index 000000000..213324efa --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/MIRACLRetrieval.json @@ -0,0 +1,48 @@ +{ + "dataset_revision": "main", + "evaluation_time": 3600, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.68426, + "map_at_1": 0.34295, + "map_at_10": 0.57322, + "map_at_100": 0.60019, + "map_at_1000": 0.60114, + "map_at_3": 0.4955, + "map_at_5": 0.54518, + "mrr_at_1": 0.6445, + "mrr_at_10": 0.7592, + "mrr_at_100": 0.76175, + "mrr_at_1000": 0.76183, + "mrr_at_3": 0.74718, + "mrr_at_5": 0.75136, + "ndcg_at_1": 0.6655, + "ndcg_at_10": 0.68426, + "ndcg_at_100": 0.74688, + "ndcg_at_1000": 0.75717, + "ndcg_at_3": 0.64003, + "ndcg_at_5": 0.66579, + "precision_at_1": 0.6655, + "precision_at_10": 0.20387, + "precision_at_100": 0.02732, + "precision_at_1000": 0.00282, + "precision_at_3": 0.46913, + "precision_at_5": 0.34371, + "recall_at_1": 0.34295, + "recall_at_10": 0.76233, + "recall_at_100": 0.95458, + "recall_at_1000": 0.9926, + "recall_at_3": 0.56624, + "recall_at_5": 0.6714 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/MintakaRetrieval.json b/results/voyageai__voyage-3/no_revision_available/MintakaRetrieval.json new file mode 100644 index 000000000..0bc5056ef --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/MintakaRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "efa78cc2f74bbcd21eff2261f9e13aebe40b814e", + "evaluation_time": 14.329999685287476, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "fr", + "languages": [ + "fra-Latn" + ], + "main_score": 0.44564, + "map_at_1": 0.2715, + "map_at_10": 0.38526, + "map_at_100": 0.39505, + "map_at_1000": 0.39562, + "map_at_20": 0.39105, + "map_at_3": 0.35313, + "map_at_5": 0.37135, + "mrr_at_1": 0.2714987714987715, + "mrr_at_10": 0.38525928525928615, + "mrr_at_100": 0.3950490749217889, + "mrr_at_1000": 0.39561902803580196, + "mrr_at_20": 0.39104548300122555, + "mrr_at_3": 0.3531258531258537, + "mrr_at_5": 0.3713486213486227, + "nauc_map_at_1000_diff1": 0.3445905808448344, + "nauc_map_at_1000_max": 0.30255518595105807, + "nauc_map_at_1000_std": -0.08428267664445518, + "nauc_map_at_100_diff1": 0.34431450927050145, + "nauc_map_at_100_max": 0.3028578220448587, + "nauc_map_at_100_std": -0.0840525200803971, + "nauc_map_at_10_diff1": 0.34297822893159025, + "nauc_map_at_10_max": 0.30461130542753806, + "nauc_map_at_10_std": -0.08458756231287945, + "nauc_map_at_1_diff1": 0.4044534696548829, + "nauc_map_at_1_max": 0.2544406615486771, + "nauc_map_at_1_std": -0.09969992897109041, + "nauc_map_at_20_diff1": 0.34259317516435195, + "nauc_map_at_20_max": 0.30335193108518843, + "nauc_map_at_20_std": -0.0847220234061382, + "nauc_map_at_3_diff1": 0.3557925401562388, + "nauc_map_at_3_max": 0.2899963987502578, + "nauc_map_at_3_std": -0.0929823360977483, + "nauc_map_at_5_diff1": 0.3452085900177116, + "nauc_map_at_5_max": 0.2969362337440696, + "nauc_map_at_5_std": -0.08996225681598671, + "nauc_mrr_at_1000_diff1": 0.3445905808448344, + "nauc_mrr_at_1000_max": 0.30255518595105807, + "nauc_mrr_at_1000_std": -0.08428267664445518, + "nauc_mrr_at_100_diff1": 0.34431450927050145, + "nauc_mrr_at_100_max": 0.3028578220448587, + "nauc_mrr_at_100_std": -0.0840525200803971, + "nauc_mrr_at_10_diff1": 0.34297822893159025, + "nauc_mrr_at_10_max": 0.30461130542753806, + "nauc_mrr_at_10_std": -0.08458756231287945, + "nauc_mrr_at_1_diff1": 0.4044534696548829, + "nauc_mrr_at_1_max": 0.2544406615486771, + "nauc_mrr_at_1_std": -0.09969992897109041, + "nauc_mrr_at_20_diff1": 0.34259317516435195, + "nauc_mrr_at_20_max": 0.30335193108518843, + "nauc_mrr_at_20_std": -0.0847220234061382, + "nauc_mrr_at_3_diff1": 0.3557925401562388, + "nauc_mrr_at_3_max": 0.2899963987502578, + "nauc_mrr_at_3_std": -0.0929823360977483, + "nauc_mrr_at_5_diff1": 0.3452085900177116, + "nauc_mrr_at_5_max": 0.2969362337440696, + "nauc_mrr_at_5_std": -0.08996225681598671, + "nauc_ndcg_at_1000_diff1": 0.3315431398582925, + "nauc_ndcg_at_1000_max": 0.3167926475916484, + "nauc_ndcg_at_1000_std": -0.07340589611256765, + "nauc_ndcg_at_100_diff1": 0.3230439716729189, + "nauc_ndcg_at_100_max": 0.3246166471509296, + "nauc_ndcg_at_100_std": -0.06285176599675081, + "nauc_ndcg_at_10_diff1": 0.31520867319213763, + "nauc_ndcg_at_10_max": 0.33315295939580325, + "nauc_ndcg_at_10_std": -0.06988770663130556, + "nauc_ndcg_at_1_diff1": 0.4044534696548829, + "nauc_ndcg_at_1_max": 0.2544406615486771, + "nauc_ndcg_at_1_std": -0.09969992897109041, + "nauc_ndcg_at_20_diff1": 0.3127018942051961, + "nauc_ndcg_at_20_max": 0.32981106066988686, + "nauc_ndcg_at_20_std": -0.06928271707494522, + "nauc_ndcg_at_3_diff1": 0.34176145843937855, + "nauc_ndcg_at_3_max": 0.3007726654233272, + "nauc_ndcg_at_3_std": -0.08985231158325264, + "nauc_ndcg_at_5_diff1": 0.3223755181069222, + "nauc_ndcg_at_5_max": 0.31346852588139434, + "nauc_ndcg_at_5_std": -0.08406711383578759, + "nauc_precision_at_1000_diff1": 0.7025463051338938, + "nauc_precision_at_1000_max": 0.7432401012288763, + "nauc_precision_at_1000_std": 0.3147535781414852, + "nauc_precision_at_100_diff1": 0.18237427106083343, + "nauc_precision_at_100_max": 0.47595605553627535, + "nauc_precision_at_100_std": 0.15757061130971567, + "nauc_precision_at_10_diff1": 0.21683235778773635, + "nauc_precision_at_10_max": 0.4381743739718916, + "nauc_precision_at_10_std": -0.012756048947345925, + "nauc_precision_at_1_diff1": 0.4044534696548829, + "nauc_precision_at_1_max": 0.2544406615486771, + "nauc_precision_at_1_std": -0.09969992897109041, + "nauc_precision_at_20_diff1": 0.1845688223337563, + "nauc_precision_at_20_max": 0.4433231597391926, + "nauc_precision_at_20_std": 0.0032667666821757308, + "nauc_precision_at_3_diff1": 0.30285537898076087, + "nauc_precision_at_3_max": 0.33082106995535576, + "nauc_precision_at_3_std": -0.08079199136560672, + "nauc_precision_at_5_diff1": 0.2534454715636124, + "nauc_precision_at_5_max": 0.36331119314036175, + "nauc_precision_at_5_std": -0.06555561286423685, + "nauc_recall_at_1000_diff1": 0.7025463051338721, + "nauc_recall_at_1000_max": 0.7432401012288703, + "nauc_recall_at_1000_std": 0.3147535781413993, + "nauc_recall_at_100_diff1": 0.18237427106083215, + "nauc_recall_at_100_max": 0.47595605553627424, + "nauc_recall_at_100_std": 0.15757061130971575, + "nauc_recall_at_10_diff1": 0.21683235778773566, + "nauc_recall_at_10_max": 0.43817437397189213, + "nauc_recall_at_10_std": -0.012756048947346497, + "nauc_recall_at_1_diff1": 0.4044534696548829, + "nauc_recall_at_1_max": 0.2544406615486771, + "nauc_recall_at_1_std": -0.09969992897109041, + "nauc_recall_at_20_diff1": 0.18456882233375677, + "nauc_recall_at_20_max": 0.44332315973919256, + "nauc_recall_at_20_std": 0.003266766682175732, + "nauc_recall_at_3_diff1": 0.3028553789807609, + "nauc_recall_at_3_max": 0.3308210699553557, + "nauc_recall_at_3_std": -0.08079199136560673, + "nauc_recall_at_5_diff1": 0.2534454715636123, + "nauc_recall_at_5_max": 0.36331119314036153, + "nauc_recall_at_5_std": -0.06555561286423682, + "ndcg_at_1": 0.2715, + "ndcg_at_10": 0.44564, + "ndcg_at_100": 0.49526, + "ndcg_at_1000": 0.51059, + "ndcg_at_20": 0.46653, + "ndcg_at_3": 0.37902, + "ndcg_at_5": 0.41197, + "precision_at_1": 0.2715, + "precision_at_10": 0.0638, + "precision_at_100": 0.00875, + "precision_at_1000": 0.001, + "precision_at_20": 0.03602, + "precision_at_3": 0.15124, + "precision_at_5": 0.1068, + "recall_at_1": 0.2715, + "recall_at_10": 0.638, + "recall_at_100": 0.87469, + "recall_at_1000": 0.9955, + "recall_at_20": 0.72031, + "recall_at_3": 0.45373, + "recall_at_5": 0.53399 + } + ] + }, + "task_name": "MintakaRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/RiaNewsRetrieval.json b/results/voyageai__voyage-3/no_revision_available/RiaNewsRetrieval.json new file mode 100644 index 000000000..631b95729 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/RiaNewsRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "82374b0bbacda6114f39ff9c5b925fa1512ca5d7", + "evaluation_time": 3749.7334699630737, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.88023, + "map_at_1": 0.792, + "map_at_10": 0.85523, + "map_at_100": 0.85647, + "map_at_1000": 0.85652, + "map_at_20": 0.85605, + "map_at_3": 0.84613, + "map_at_5": 0.85243, + "mrr_at_1": 0.792, + "mrr_at_10": 0.8552301587301563, + "mrr_at_100": 0.8564682050153509, + "mrr_at_1000": 0.8565153703077655, + "mrr_at_20": 0.8560518725173144, + "mrr_at_3": 0.8461333333333306, + "mrr_at_5": 0.8524283333333298, + "nauc_map_at_1000_diff1": 0.8058609843634124, + "nauc_map_at_1000_max": 0.4363552923730781, + "nauc_map_at_1000_std": -0.23601858224868305, + "nauc_map_at_100_diff1": 0.8058362309457052, + "nauc_map_at_100_max": 0.4364831267744464, + "nauc_map_at_100_std": -0.23591946426228116, + "nauc_map_at_10_diff1": 0.8055080046026662, + "nauc_map_at_10_max": 0.43729372146191153, + "nauc_map_at_10_std": -0.23639522781194885, + "nauc_map_at_1_diff1": 0.8264070585499156, + "nauc_map_at_1_max": 0.38811728395061695, + "nauc_map_at_1_std": -0.26235173556602226, + "nauc_map_at_20_diff1": 0.8057311837501498, + "nauc_map_at_20_max": 0.4370977540355562, + "nauc_map_at_20_std": -0.23592130284282123, + "nauc_map_at_3_diff1": 0.8031450748436693, + "nauc_map_at_3_max": 0.4383451863901369, + "nauc_map_at_3_std": -0.2438698516440503, + "nauc_map_at_5_diff1": 0.8042857001260755, + "nauc_map_at_5_max": 0.4377870176916221, + "nauc_map_at_5_std": -0.2397316166439884, + "nauc_mrr_at_1000_diff1": 0.8058609843634124, + "nauc_mrr_at_1000_max": 0.4363552923730781, + "nauc_mrr_at_1000_std": -0.23601858224868305, + "nauc_mrr_at_100_diff1": 0.8058362309457052, + "nauc_mrr_at_100_max": 0.4364831267744464, + "nauc_mrr_at_100_std": -0.23591946426228116, + "nauc_mrr_at_10_diff1": 0.8055080046026662, + "nauc_mrr_at_10_max": 0.43729372146191153, + "nauc_mrr_at_10_std": -0.23639522781194885, + "nauc_mrr_at_1_diff1": 0.8264070585499156, + "nauc_mrr_at_1_max": 0.38811728395061695, + "nauc_mrr_at_1_std": -0.26235173556602226, + "nauc_mrr_at_20_diff1": 0.8057311837501498, + "nauc_mrr_at_20_max": 0.4370977540355562, + "nauc_mrr_at_20_std": -0.23592130284282123, + "nauc_mrr_at_3_diff1": 0.8031450748436693, + "nauc_mrr_at_3_max": 0.4383451863901369, + "nauc_mrr_at_3_std": -0.2438698516440503, + "nauc_mrr_at_5_diff1": 0.8042857001260755, + "nauc_mrr_at_5_max": 0.4377870176916221, + "nauc_mrr_at_5_std": -0.2397316166439884, + "nauc_ndcg_at_1000_diff1": 0.801047240084407, + "nauc_ndcg_at_1000_max": 0.45073042632830024, + "nauc_ndcg_at_1000_std": -0.2176040752673121, + "nauc_ndcg_at_100_diff1": 0.7998976138065916, + "nauc_ndcg_at_100_max": 0.4550502760742729, + "nauc_ndcg_at_100_std": -0.2126898518358668, + "nauc_ndcg_at_10_diff1": 0.7981901240929227, + "nauc_ndcg_at_10_max": 0.46032964206021987, + "nauc_ndcg_at_10_std": -0.21487407955050147, + "nauc_ndcg_at_1_diff1": 0.8264070585499156, + "nauc_ndcg_at_1_max": 0.38811728395061695, + "nauc_ndcg_at_1_std": -0.26235173556602226, + "nauc_ndcg_at_20_diff1": 0.7989218991311996, + "nauc_ndcg_at_20_max": 0.460040515848982, + "nauc_ndcg_at_20_std": -0.21235191956153937, + "nauc_ndcg_at_3_diff1": 0.7936211390114346, + "nauc_ndcg_at_3_max": 0.45911701102160507, + "nauc_ndcg_at_3_std": -0.23552117369836006, + "nauc_ndcg_at_5_diff1": 0.7950705867889006, + "nauc_ndcg_at_5_max": 0.4602529084093987, + "nauc_ndcg_at_5_std": -0.22547850505462758, + "nauc_precision_at_1000_diff1": 0.6845178749465818, + "nauc_precision_at_1000_max": 0.8664245359160706, + "nauc_precision_at_1000_std": 0.6852775008308662, + "nauc_precision_at_100_diff1": 0.6896335053937658, + "nauc_precision_at_100_max": 0.8388856590854262, + "nauc_precision_at_100_std": 0.40548420626112475, + "nauc_precision_at_10_diff1": 0.7378415597593695, + "nauc_precision_at_10_max": 0.666984297524189, + "nauc_precision_at_10_std": -0.0038041091626916006, + "nauc_precision_at_1_diff1": 0.8264070585499156, + "nauc_precision_at_1_max": 0.38811728395061695, + "nauc_precision_at_1_std": -0.26235173556602226, + "nauc_precision_at_20_diff1": 0.7248755301493693, + "nauc_precision_at_20_max": 0.7397504456328002, + "nauc_precision_at_20_std": 0.10836121282368759, + "nauc_precision_at_3_diff1": 0.7504592084286424, + "nauc_precision_at_3_max": 0.5533599174749972, + "nauc_precision_at_3_std": -0.1972853187854454, + "nauc_precision_at_5_diff1": 0.7387582423150152, + "nauc_precision_at_5_max": 0.5995997365354435, + "nauc_precision_at_5_std": -0.13294899355090736, + "nauc_recall_at_1000_diff1": 0.6845178749465715, + "nauc_recall_at_1000_max": 0.8664245359160618, + "nauc_recall_at_1000_std": 0.6852775008308443, + "nauc_recall_at_100_diff1": 0.6896335053937723, + "nauc_recall_at_100_max": 0.8388856590854267, + "nauc_recall_at_100_std": 0.40548420626112575, + "nauc_recall_at_10_diff1": 0.7378415597593694, + "nauc_recall_at_10_max": 0.6669842975241845, + "nauc_recall_at_10_std": -0.003804109162691592, + "nauc_recall_at_1_diff1": 0.8264070585499156, + "nauc_recall_at_1_max": 0.38811728395061695, + "nauc_recall_at_1_std": -0.26235173556602226, + "nauc_recall_at_20_diff1": 0.724875530149363, + "nauc_recall_at_20_max": 0.7397504456327969, + "nauc_recall_at_20_std": 0.10836121282369135, + "nauc_recall_at_3_diff1": 0.7504592084286398, + "nauc_recall_at_3_max": 0.5533599174749954, + "nauc_recall_at_3_std": -0.19728531878544667, + "nauc_recall_at_5_diff1": 0.7387582423150151, + "nauc_recall_at_5_max": 0.5995997365354425, + "nauc_recall_at_5_std": -0.13294899355090717, + "ndcg_at_1": 0.792, + "ndcg_at_10": 0.88023, + "ndcg_at_100": 0.88621, + "ndcg_at_1000": 0.88748, + "ndcg_at_20": 0.88323, + "ndcg_at_3": 0.86225, + "ndcg_at_5": 0.8735, + "precision_at_1": 0.792, + "precision_at_10": 0.09562, + "precision_at_100": 0.00984, + "precision_at_1000": 0.00099, + "precision_at_20": 0.04841, + "precision_at_3": 0.3028, + "precision_at_5": 0.1871, + "recall_at_1": 0.792, + "recall_at_10": 0.9562, + "recall_at_100": 0.9841, + "recall_at_1000": 0.9941, + "recall_at_20": 0.9681, + "recall_at_3": 0.9084, + "recall_at_5": 0.9355 + } + ] + }, + "task_name": "RiaNewsRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/RuBQRetrieval.json b/results/voyageai__voyage-3/no_revision_available/RuBQRetrieval.json new file mode 100644 index 000000000..f796941d4 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/RuBQRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b", + "evaluation_time": 191.39519834518433, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.7154, + "map_at_1": 0.39746, + "map_at_10": 0.62885, + "map_at_100": 0.63674, + "map_at_1000": 0.63694, + "map_at_20": 0.63442, + "map_at_3": 0.56579, + "map_at_5": 0.60541, + "mrr_at_1": 0.5709219858156028, + "mrr_at_10": 0.7054385248977454, + "mrr_at_100": 0.707583091282195, + "mrr_at_1000": 0.7076248632748341, + "mrr_at_20": 0.7070120199048495, + "mrr_at_3": 0.6804570527974793, + "mrr_at_5": 0.6971532702915695, + "nauc_map_at_1000_diff1": 0.36535050094336546, + "nauc_map_at_1000_max": 0.22533621131373757, + "nauc_map_at_1000_std": -0.12282999401265435, + "nauc_map_at_100_diff1": 0.36519331026833723, + "nauc_map_at_100_max": 0.22547091937546865, + "nauc_map_at_100_std": -0.12264851002155382, + "nauc_map_at_10_diff1": 0.36278484431082525, + "nauc_map_at_10_max": 0.22537047663814344, + "nauc_map_at_10_std": -0.12492096523016746, + "nauc_map_at_1_diff1": 0.43074214159348373, + "nauc_map_at_1_max": 0.13518669796449648, + "nauc_map_at_1_std": -0.13207795049044935, + "nauc_map_at_20_diff1": 0.3635621750288924, + "nauc_map_at_20_max": 0.22653022222612731, + "nauc_map_at_20_std": -0.12199888197724695, + "nauc_map_at_3_diff1": 0.3662924836584811, + "nauc_map_at_3_max": 0.1903890468202349, + "nauc_map_at_3_std": -0.14130213371426947, + "nauc_map_at_5_diff1": 0.363779801750429, + "nauc_map_at_5_max": 0.20755267857588094, + "nauc_map_at_5_std": -0.13985667709151006, + "nauc_mrr_at_1000_diff1": 0.38911106631929715, + "nauc_mrr_at_1000_max": 0.24637130388809572, + "nauc_mrr_at_1000_std": -0.13431778270357442, + "nauc_mrr_at_100_diff1": 0.3891248266638641, + "nauc_mrr_at_100_max": 0.24646737414034714, + "nauc_mrr_at_100_std": -0.13423491703965626, + "nauc_mrr_at_10_diff1": 0.3877980932179414, + "nauc_mrr_at_10_max": 0.24874684149268705, + "nauc_mrr_at_10_std": -0.13358184773767678, + "nauc_mrr_at_1_diff1": 0.42934332909403283, + "nauc_mrr_at_1_max": 0.21626517050701216, + "nauc_mrr_at_1_std": -0.1368961595546419, + "nauc_mrr_at_20_diff1": 0.38853988627089153, + "nauc_mrr_at_20_max": 0.2470077465424185, + "nauc_mrr_at_20_std": -0.13438376482165698, + "nauc_mrr_at_3_diff1": 0.3790357711646287, + "nauc_mrr_at_3_max": 0.24443245157635332, + "nauc_mrr_at_3_std": -0.13928285370253693, + "nauc_mrr_at_5_diff1": 0.3830195516695417, + "nauc_mrr_at_5_max": 0.24600195262744345, + "nauc_mrr_at_5_std": -0.1377296788149686, + "nauc_ndcg_at_1000_diff1": 0.3654550642837905, + "nauc_ndcg_at_1000_max": 0.24977818481368136, + "nauc_ndcg_at_1000_std": -0.11171866287544231, + "nauc_ndcg_at_100_diff1": 0.3627885973400763, + "nauc_ndcg_at_100_max": 0.2540863296492309, + "nauc_ndcg_at_100_std": -0.106044168629434, + "nauc_ndcg_at_10_diff1": 0.35096455570005025, + "nauc_ndcg_at_10_max": 0.25904408489384795, + "nauc_ndcg_at_10_std": -0.11231941860635111, + "nauc_ndcg_at_1_diff1": 0.43088130343065106, + "nauc_ndcg_at_1_max": 0.21510104869883895, + "nauc_ndcg_at_1_std": -0.13549444906985425, + "nauc_ndcg_at_20_diff1": 0.3545787921138897, + "nauc_ndcg_at_20_max": 0.26025655241696527, + "nauc_ndcg_at_20_std": -0.10589136947108509, + "nauc_ndcg_at_3_diff1": 0.34762474676864646, + "nauc_ndcg_at_3_max": 0.21022940208214574, + "nauc_ndcg_at_3_std": -0.14404791985180565, + "nauc_ndcg_at_5_diff1": 0.3490547187850728, + "nauc_ndcg_at_5_max": 0.22767411678163357, + "nauc_ndcg_at_5_std": -0.14089952069862344, + "nauc_precision_at_1000_diff1": -0.13232879513514234, + "nauc_precision_at_1000_max": 0.09355889342206758, + "nauc_precision_at_1000_std": 0.0822007647090045, + "nauc_precision_at_100_diff1": -0.13217206895862482, + "nauc_precision_at_100_max": 0.12043135094266728, + "nauc_precision_at_100_std": 0.10317721326923542, + "nauc_precision_at_10_diff1": -0.09862445431893314, + "nauc_precision_at_10_max": 0.1823338830128631, + "nauc_precision_at_10_std": 0.07474775545344178, + "nauc_precision_at_1_diff1": 0.43088130343065106, + "nauc_precision_at_1_max": 0.21510104869883895, + "nauc_precision_at_1_std": -0.13549444906985425, + "nauc_precision_at_20_diff1": -0.12292211947863191, + "nauc_precision_at_20_max": 0.16256785883491567, + "nauc_precision_at_20_std": 0.10020332732575671, + "nauc_precision_at_3_diff1": 0.05193643654624038, + "nauc_precision_at_3_max": 0.2000742936672456, + "nauc_precision_at_3_std": -0.05043967599762624, + "nauc_precision_at_5_diff1": -0.02724150199312052, + "nauc_precision_at_5_max": 0.18084717163046882, + "nauc_precision_at_5_std": -0.011272220130325963, + "nauc_recall_at_1000_diff1": 0.41136542896118, + "nauc_recall_at_1000_max": 0.8187930583599274, + "nauc_recall_at_1000_std": 0.7221196937644317, + "nauc_recall_at_100_diff1": 0.20580329016082807, + "nauc_recall_at_100_max": 0.5796074461845855, + "nauc_recall_at_100_std": 0.40041012072290244, + "nauc_recall_at_10_diff1": 0.19356862901672925, + "nauc_recall_at_10_max": 0.3671380957477402, + "nauc_recall_at_10_std": -0.014235689499124814, + "nauc_recall_at_1_diff1": 0.43074214159348373, + "nauc_recall_at_1_max": 0.13518669796449648, + "nauc_recall_at_1_std": -0.13207795049044935, + "nauc_recall_at_20_diff1": 0.1647792249084328, + "nauc_recall_at_20_max": 0.43350512846511413, + "nauc_recall_at_20_std": 0.07965348964474303, + "nauc_recall_at_3_diff1": 0.2597336249526316, + "nauc_recall_at_3_max": 0.18774826870491484, + "nauc_recall_at_3_std": -0.13516827245270768, + "nauc_recall_at_5_diff1": 0.23652590765466186, + "nauc_recall_at_5_max": 0.2281970903613387, + "nauc_recall_at_5_std": -0.13181240705041686, + "ndcg_at_1": 0.57033, + "ndcg_at_10": 0.7154, + "ndcg_at_100": 0.73872, + "ndcg_at_1000": 0.74178, + "ndcg_at_20": 0.72857, + "ndcg_at_3": 0.63012, + "ndcg_at_5": 0.67643, + "precision_at_1": 0.57033, + "precision_at_10": 0.14545, + "precision_at_100": 0.01628, + "precision_at_1000": 0.00167, + "precision_at_20": 0.07689, + "precision_at_3": 0.35225, + "precision_at_5": 0.25189, + "recall_at_1": 0.39746, + "recall_at_10": 0.88632, + "recall_at_100": 0.97404, + "recall_at_1000": 0.9933, + "recall_at_20": 0.92974, + "recall_at_3": 0.67801, + "recall_at_5": 0.78556 + } + ] + }, + "task_name": "RuBQRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/StackOverflowQA.json b/results/voyageai__voyage-3/no_revision_available/StackOverflowQA.json new file mode 100644 index 000000000..265f9b9a0 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/StackOverflowQA.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "db8f169f3894c14a00251061f957b2063eef2bd5", + "evaluation_time": 149.1224341392517, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn" + ], + "main_score": 0.94335, + "map_at_1": 0.8992, + "map_at_10": 0.93071, + "map_at_100": 0.93136, + "map_at_1000": 0.93138, + "map_at_20": 0.93112, + "map_at_3": 0.92695, + "map_at_5": 0.92885, + "mrr_at_1": 0.899197592778335, + "mrr_at_10": 0.93071157918199, + "mrr_at_100": 0.9313598218736854, + "mrr_at_1000": 0.9313777630574932, + "mrr_at_20": 0.9311248627487783, + "mrr_at_3": 0.926947509194249, + "mrr_at_5": 0.9288532263457033, + "nauc_map_at_1000_diff1": 0.9196416675624052, + "nauc_map_at_1000_max": 0.8401189007482858, + "nauc_map_at_1000_std": 0.03780961915661576, + "nauc_map_at_100_diff1": 0.9196357452142138, + "nauc_map_at_100_max": 0.8401354997660466, + "nauc_map_at_100_std": 0.03777433459036646, + "nauc_map_at_10_diff1": 0.9199945459915102, + "nauc_map_at_10_max": 0.8406318173755345, + "nauc_map_at_10_std": 0.036454204177167604, + "nauc_map_at_1_diff1": 0.915530471032786, + "nauc_map_at_1_max": 0.8214154586263914, + "nauc_map_at_1_std": 0.02152584147986912, + "nauc_map_at_20_diff1": 0.9197348261184244, + "nauc_map_at_20_max": 0.840471837374836, + "nauc_map_at_20_std": 0.03811994590706359, + "nauc_map_at_3_diff1": 0.9193311477189411, + "nauc_map_at_3_max": 0.8435023614357626, + "nauc_map_at_3_std": 0.044332277575193646, + "nauc_map_at_5_diff1": 0.9196868636690444, + "nauc_map_at_5_max": 0.8418046807138682, + "nauc_map_at_5_std": 0.03875110193708013, + "nauc_mrr_at_1000_diff1": 0.9196416675624052, + "nauc_mrr_at_1000_max": 0.8401189007482858, + "nauc_mrr_at_1000_std": 0.03780961915661576, + "nauc_mrr_at_100_diff1": 0.9196357452142138, + "nauc_mrr_at_100_max": 0.8401354997660466, + "nauc_mrr_at_100_std": 0.03777433459036646, + "nauc_mrr_at_10_diff1": 0.9199945459915102, + "nauc_mrr_at_10_max": 0.8406318173755345, + "nauc_mrr_at_10_std": 0.036454204177167604, + "nauc_mrr_at_1_diff1": 0.915530471032786, + "nauc_mrr_at_1_max": 0.8214154586263914, + "nauc_mrr_at_1_std": 0.02152584147986912, + "nauc_mrr_at_20_diff1": 0.9197348261184244, + "nauc_mrr_at_20_max": 0.840471837374836, + "nauc_mrr_at_20_std": 0.03811994590706359, + "nauc_mrr_at_3_diff1": 0.9193311477189411, + "nauc_mrr_at_3_max": 0.8435023614357626, + "nauc_mrr_at_3_std": 0.044332277575193646, + "nauc_mrr_at_5_diff1": 0.9196868636690444, + "nauc_mrr_at_5_max": 0.8418046807138682, + "nauc_mrr_at_5_std": 0.03875110193708013, + "nauc_ndcg_at_1000_diff1": 0.9203963244509442, + "nauc_ndcg_at_1000_max": 0.8431280073564209, + "nauc_ndcg_at_1000_std": 0.039023798697103186, + "nauc_ndcg_at_100_diff1": 0.9203455498978669, + "nauc_ndcg_at_100_max": 0.8436789452065547, + "nauc_ndcg_at_100_std": 0.038075801892110496, + "nauc_ndcg_at_10_diff1": 0.9223456883943264, + "nauc_ndcg_at_10_max": 0.8470882265456543, + "nauc_ndcg_at_10_std": 0.034852282206277924, + "nauc_ndcg_at_1_diff1": 0.915530471032786, + "nauc_ndcg_at_1_max": 0.8214154586263914, + "nauc_ndcg_at_1_std": 0.02152584147986912, + "nauc_ndcg_at_20_diff1": 0.9212625651205985, + "nauc_ndcg_at_20_max": 0.8464059353805045, + "nauc_ndcg_at_20_std": 0.040912876568650725, + "nauc_ndcg_at_3_diff1": 0.9206043211724404, + "nauc_ndcg_at_3_max": 0.8533390297126524, + "nauc_ndcg_at_3_std": 0.052891544192552646, + "nauc_ndcg_at_5_diff1": 0.9214107246432048, + "nauc_ndcg_at_5_max": 0.8501904029805435, + "nauc_ndcg_at_5_std": 0.04113408676331023, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": 0.919602519460112, + "nauc_precision_at_100_max": 0.8947107055115057, + "nauc_precision_at_100_std": -0.04573117971896781, + "nauc_precision_at_10_diff1": 0.9489104410070267, + "nauc_precision_at_10_max": 0.9101422104105925, + "nauc_precision_at_10_std": -0.0008948124590673897, + "nauc_precision_at_1_diff1": 0.915530471032786, + "nauc_precision_at_1_max": 0.8214154586263914, + "nauc_precision_at_1_std": 0.02152584147986912, + "nauc_precision_at_20_diff1": 0.9421138140112923, + "nauc_precision_at_20_max": 0.9248472055215124, + "nauc_precision_at_20_std": 0.07535917624306562, + "nauc_precision_at_3_diff1": 0.9269767992725875, + "nauc_precision_at_3_max": 0.9043922572538454, + "nauc_precision_at_3_std": 0.09641827660136111, + "nauc_precision_at_5_diff1": 0.932425544075469, + "nauc_precision_at_5_max": 0.902785464953265, + "nauc_precision_at_5_std": 0.05069371751256923, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.9196025194601508, + "nauc_recall_at_100_max": 0.8947107055115172, + "nauc_recall_at_100_std": -0.04573117971897193, + "nauc_recall_at_10_diff1": 0.9489104410070297, + "nauc_recall_at_10_max": 0.9101422104105926, + "nauc_recall_at_10_std": -0.0008948124590565184, + "nauc_recall_at_1_diff1": 0.915530471032786, + "nauc_recall_at_1_max": 0.8214154586263914, + "nauc_recall_at_1_std": 0.02152584147986912, + "nauc_recall_at_20_diff1": 0.9421138140112981, + "nauc_recall_at_20_max": 0.9248472055215184, + "nauc_recall_at_20_std": 0.07535917624305702, + "nauc_recall_at_3_diff1": 0.9269767992725906, + "nauc_recall_at_3_max": 0.90439225725385, + "nauc_recall_at_3_std": 0.09641827660135817, + "nauc_recall_at_5_diff1": 0.9324255440754683, + "nauc_recall_at_5_max": 0.9027854649532716, + "nauc_recall_at_5_std": 0.05069371751257517, + "ndcg_at_1": 0.8992, + "ndcg_at_10": 0.94335, + "ndcg_at_100": 0.94633, + "ndcg_at_1000": 0.94689, + "ndcg_at_20": 0.94478, + "ndcg_at_3": 0.93545, + "ndcg_at_5": 0.93892, + "precision_at_1": 0.8992, + "precision_at_10": 0.09819, + "precision_at_100": 0.00995, + "precision_at_1000": 0.001, + "precision_at_20": 0.04937, + "precision_at_3": 0.31996, + "precision_at_5": 0.19368, + "recall_at_1": 0.8992, + "recall_at_10": 0.98195, + "recall_at_100": 0.99549, + "recall_at_1000": 1.0, + "recall_at_20": 0.98746, + "recall_at_3": 0.95988, + "recall_at_5": 0.96841 + } + ] + }, + "task_name": "StackOverflowQA" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/SyntecRetrieval.json b/results/voyageai__voyage-3/no_revision_available/SyntecRetrieval.json new file mode 100644 index 000000000..6951672ea --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/SyntecRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "19661ccdca4dfc2d15122d776b61685f48c68ca9", + "evaluation_time": 1.546055555343628, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "fra-Latn" + ], + "main_score": 0.87537, + "map_at_1": 0.74, + "map_at_10": 0.83733, + "map_at_100": 0.83824, + "map_at_1000": 0.83824, + "map_at_20": 0.83824, + "map_at_3": 0.82833, + "map_at_5": 0.83483, + "mrr_at_1": 0.74, + "mrr_at_10": 0.8373333333333334, + "mrr_at_100": 0.8382424242424242, + "mrr_at_1000": 0.8382424242424242, + "mrr_at_20": 0.8382424242424242, + "mrr_at_3": 0.8283333333333333, + "mrr_at_5": 0.8348333333333333, + "nauc_map_at_1000_diff1": 0.6549528255797319, + "nauc_map_at_1000_max": 0.44936213485365795, + "nauc_map_at_1000_std": -0.02369955206406287, + "nauc_map_at_100_diff1": 0.6549528255797319, + "nauc_map_at_100_max": 0.44936213485365795, + "nauc_map_at_100_std": -0.02369955206406287, + "nauc_map_at_10_diff1": 0.6498478127981085, + "nauc_map_at_10_max": 0.4500095711895563, + "nauc_map_at_10_std": -0.02134084433755593, + "nauc_map_at_1_diff1": 0.6583471878674829, + "nauc_map_at_1_max": 0.40791127691496654, + "nauc_map_at_1_std": -0.04612546125461295, + "nauc_map_at_20_diff1": 0.6549528255797319, + "nauc_map_at_20_max": 0.44936213485365795, + "nauc_map_at_20_std": -0.02369955206406287, + "nauc_map_at_3_diff1": 0.6601337479319891, + "nauc_map_at_3_max": 0.45964141202734793, + "nauc_map_at_3_std": -0.020675118100818794, + "nauc_map_at_5_diff1": 0.6535137389235357, + "nauc_map_at_5_max": 0.45547367906565145, + "nauc_map_at_5_std": -0.027165350645945642, + "nauc_mrr_at_1000_diff1": 0.6549528255797319, + "nauc_mrr_at_1000_max": 0.44936213485365795, + "nauc_mrr_at_1000_std": -0.02369955206406287, + "nauc_mrr_at_100_diff1": 0.6549528255797319, + "nauc_mrr_at_100_max": 0.44936213485365795, + "nauc_mrr_at_100_std": -0.02369955206406287, + "nauc_mrr_at_10_diff1": 0.6498478127981085, + "nauc_mrr_at_10_max": 0.4500095711895563, + "nauc_mrr_at_10_std": -0.02134084433755593, + "nauc_mrr_at_1_diff1": 0.6583471878674829, + "nauc_mrr_at_1_max": 0.40791127691496654, + "nauc_mrr_at_1_std": -0.04612546125461295, + "nauc_mrr_at_20_diff1": 0.6549528255797319, + "nauc_mrr_at_20_max": 0.44936213485365795, + "nauc_mrr_at_20_std": -0.02369955206406287, + "nauc_mrr_at_3_diff1": 0.6601337479319891, + "nauc_mrr_at_3_max": 0.45964141202734793, + "nauc_mrr_at_3_std": -0.020675118100818794, + "nauc_mrr_at_5_diff1": 0.6535137389235357, + "nauc_mrr_at_5_max": 0.45547367906565145, + "nauc_mrr_at_5_std": -0.027165350645945642, + "nauc_ndcg_at_1000_diff1": 0.6535642293042059, + "nauc_ndcg_at_1000_max": 0.4532621270764946, + "nauc_ndcg_at_1000_std": -0.02213807406985956, + "nauc_ndcg_at_100_diff1": 0.6535642293042059, + "nauc_ndcg_at_100_max": 0.4532621270764946, + "nauc_ndcg_at_100_std": -0.02213807406985956, + "nauc_ndcg_at_10_diff1": 0.6332260561048726, + "nauc_ndcg_at_10_max": 0.4557496747149011, + "nauc_ndcg_at_10_std": -0.012763754147406454, + "nauc_ndcg_at_1_diff1": 0.6583471878674829, + "nauc_ndcg_at_1_max": 0.40791127691496654, + "nauc_ndcg_at_1_std": -0.04612546125461295, + "nauc_ndcg_at_20_diff1": 0.6535642293042059, + "nauc_ndcg_at_20_max": 0.4532621270764946, + "nauc_ndcg_at_20_std": -0.02213807406985956, + "nauc_ndcg_at_3_diff1": 0.6607487544008724, + "nauc_ndcg_at_3_max": 0.48110743978074033, + "nauc_ndcg_at_3_std": -0.014158387109161407, + "nauc_ndcg_at_5_diff1": 0.645694771246683, + "nauc_ndcg_at_5_max": 0.4727226058844881, + "nauc_ndcg_at_5_std": -0.03163242602782032, + "nauc_precision_at_1000_diff1": NaN, + "nauc_precision_at_1000_max": NaN, + "nauc_precision_at_1000_std": NaN, + "nauc_precision_at_100_diff1": NaN, + "nauc_precision_at_100_max": NaN, + "nauc_precision_at_100_std": NaN, + "nauc_precision_at_10_diff1": -0.1713352007469681, + "nauc_precision_at_10_max": 0.5541549953314738, + "nauc_precision_at_10_std": 0.35807656395891135, + "nauc_precision_at_1_diff1": 0.6583471878674829, + "nauc_precision_at_1_max": 0.40791127691496654, + "nauc_precision_at_1_std": -0.04612546125461295, + "nauc_precision_at_20_diff1": 1.0, + "nauc_precision_at_20_max": 1.0, + "nauc_precision_at_20_std": 1.0, + "nauc_precision_at_3_diff1": 0.6651104886398984, + "nauc_precision_at_3_max": 0.6374105197634568, + "nauc_precision_at_3_std": 0.02956738250855897, + "nauc_precision_at_5_diff1": 0.5224089635854351, + "nauc_precision_at_5_max": 0.7027699968876432, + "nauc_precision_at_5_std": -0.1272953625894841, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": NaN, + "nauc_recall_at_100_max": NaN, + "nauc_recall_at_100_std": NaN, + "nauc_recall_at_10_diff1": -0.17133520074696149, + "nauc_recall_at_10_max": 0.554154995331463, + "nauc_recall_at_10_std": 0.35807656395891085, + "nauc_recall_at_1_diff1": 0.6583471878674829, + "nauc_recall_at_1_max": 0.40791127691496654, + "nauc_recall_at_1_std": -0.04612546125461295, + "nauc_recall_at_20_diff1": NaN, + "nauc_recall_at_20_max": NaN, + "nauc_recall_at_20_std": NaN, + "nauc_recall_at_3_diff1": 0.6651104886398995, + "nauc_recall_at_3_max": 0.6374105197634621, + "nauc_recall_at_3_std": 0.02956738250856233, + "nauc_recall_at_5_diff1": 0.5224089635854358, + "nauc_recall_at_5_max": 0.7027699968876437, + "nauc_recall_at_5_std": -0.12729536258947716, + "ndcg_at_1": 0.74, + "ndcg_at_10": 0.87537, + "ndcg_at_100": 0.87816, + "ndcg_at_1000": 0.87816, + "ndcg_at_20": 0.87816, + "ndcg_at_3": 0.85702, + "ndcg_at_5": 0.86906, + "precision_at_1": 0.74, + "precision_at_10": 0.099, + "precision_at_100": 0.01, + "precision_at_1000": 0.001, + "precision_at_20": 0.05, + "precision_at_3": 0.31333, + "precision_at_5": 0.194, + "recall_at_1": 0.74, + "recall_at_10": 0.99, + "recall_at_100": 1.0, + "recall_at_1000": 1.0, + "recall_at_20": 1.0, + "recall_at_3": 0.94, + "recall_at_5": 0.97 + } + ] + }, + "task_name": "SyntecRetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/SyntheticText2SQL.json b/results/voyageai__voyage-3/no_revision_available/SyntheticText2SQL.json new file mode 100644 index 000000000..5509d5e69 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/SyntheticText2SQL.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "686b87296c3a0191b5d9415a00526c62db9fce09", + "evaluation_time": 520.2923486232758, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "default", + "languages": [ + "eng-Latn", + "sql-Code" + ], + "main_score": 0.57563, + "map_at_1": 0.0523, + "map_at_10": 0.45091, + "map_at_100": 0.45338, + "map_at_1000": 0.45342, + "map_at_20": 0.45248, + "map_at_3": 0.43221, + "map_at_5": 0.44492, + "mrr_at_1": 0.39463339600068364, + "mrr_at_10": 0.6319478694457349, + "mrr_at_100": 0.6343066289458856, + "mrr_at_1000": 0.6343426910763931, + "mrr_at_20": 0.6334201305533657, + "mrr_at_3": 0.6165612715775114, + "mrr_at_5": 0.6267133823278094, + "nauc_map_at_1000_diff1": -0.32572374624014205, + "nauc_map_at_1000_max": 0.2814209140899471, + "nauc_map_at_1000_std": -0.25533221846289395, + "nauc_map_at_100_diff1": -0.3259920429545265, + "nauc_map_at_100_max": 0.28154643153585107, + "nauc_map_at_100_std": -0.25528364463578435, + "nauc_map_at_10_diff1": -0.34030333840853827, + "nauc_map_at_10_max": 0.28680178921332095, + "nauc_map_at_10_std": -0.25592608528887556, + "nauc_map_at_1_diff1": 0.7903694326100982, + "nauc_map_at_1_max": 0.05856215046883633, + "nauc_map_at_1_std": -0.12094234457399146, + "nauc_map_at_20_diff1": -0.3314546256799683, + "nauc_map_at_20_max": 0.2832520805112207, + "nauc_map_at_20_std": -0.2555802494800351, + "nauc_map_at_3_diff1": -0.43064629133787735, + "nauc_map_at_3_max": 0.2949436545462884, + "nauc_map_at_3_std": -0.27609207737872604, + "nauc_map_at_5_diff1": -0.37293788089146857, + "nauc_map_at_5_max": 0.29114015451440706, + "nauc_map_at_5_std": -0.2660913388462142, + "nauc_mrr_at_1000_diff1": -0.4401099889784779, + "nauc_mrr_at_1000_max": 0.21043554872864134, + "nauc_mrr_at_1000_std": -0.17575439716298297, + "nauc_mrr_at_100_diff1": -0.4402076941034323, + "nauc_mrr_at_100_max": 0.21049900538862248, + "nauc_mrr_at_100_std": -0.1757472068793206, + "nauc_mrr_at_10_diff1": -0.4455754123068381, + "nauc_mrr_at_10_max": 0.213729723266942, + "nauc_mrr_at_10_std": -0.1768848431007586, + "nauc_mrr_at_1_diff1": -0.2387312145408484, + "nauc_mrr_at_1_max": 0.12938743086092155, + "nauc_mrr_at_1_std": -0.12051920142489922, + "nauc_mrr_at_20_diff1": -0.4423383506834488, + "nauc_mrr_at_20_max": 0.2115791333342484, + "nauc_mrr_at_20_std": -0.17617377140713822, + "nauc_mrr_at_3_diff1": -0.4777229143437527, + "nauc_mrr_at_3_max": 0.22257272716572032, + "nauc_mrr_at_3_std": -0.19127922374874506, + "nauc_mrr_at_5_diff1": -0.45664620139489176, + "nauc_mrr_at_5_max": 0.21844941290336956, + "nauc_mrr_at_5_std": -0.1821414050426622, + "nauc_ndcg_at_1000_diff1": -0.38994752474297506, + "nauc_ndcg_at_1000_max": 0.2995642721532435, + "nauc_ndcg_at_1000_std": -0.26127928929175104, + "nauc_ndcg_at_100_diff1": -0.39721105534952766, + "nauc_ndcg_at_100_max": 0.30326977615220696, + "nauc_ndcg_at_100_std": -0.2591096041599029, + "nauc_ndcg_at_10_diff1": -0.4650956734309437, + "nauc_ndcg_at_10_max": 0.3275502649073647, + "nauc_ndcg_at_10_std": -0.26476831083436125, + "nauc_ndcg_at_1_diff1": 0.7903694326100982, + "nauc_ndcg_at_1_max": 0.05856215046883633, + "nauc_ndcg_at_1_std": -0.12094234457399146, + "nauc_ndcg_at_20_diff1": -0.43490207081625315, + "nauc_ndcg_at_20_max": 0.3151844593732661, + "nauc_ndcg_at_20_std": -0.26352574758199415, + "nauc_ndcg_at_3_diff1": -0.6070089547078736, + "nauc_ndcg_at_3_max": 0.33555126776812944, + "nauc_ndcg_at_3_std": -0.3026889982713927, + "nauc_ndcg_at_5_diff1": -0.5304497322466718, + "nauc_ndcg_at_5_max": 0.3341133574616088, + "nauc_ndcg_at_5_std": -0.28860151343233087, + "nauc_precision_at_1000_diff1": -0.5636061826822646, + "nauc_precision_at_1000_max": 1.0, + "nauc_precision_at_1000_std": 1.0, + "nauc_precision_at_100_diff1": -1.0972131138257208, + "nauc_precision_at_100_max": 0.6788457789804003, + "nauc_precision_at_100_std": -0.005468504958678187, + "nauc_precision_at_10_diff1": -1.0232817354629145, + "nauc_precision_at_10_max": 0.538993177581218, + "nauc_precision_at_10_std": -0.2854490613119887, + "nauc_precision_at_1_diff1": 0.7903694326100982, + "nauc_precision_at_1_max": 0.05856215046883633, + "nauc_precision_at_1_std": -0.12094234457399146, + "nauc_precision_at_20_diff1": -1.036891106974817, + "nauc_precision_at_20_max": 0.5361309760012369, + "nauc_precision_at_20_std": -0.2835267458980144, + "nauc_precision_at_3_diff1": -1.033270818345793, + "nauc_precision_at_3_max": 0.4363810748801292, + "nauc_precision_at_3_std": -0.36871033246815993, + "nauc_precision_at_5_diff1": -1.0189881929347981, + "nauc_precision_at_5_max": 0.47559438109485497, + "nauc_precision_at_5_std": -0.35994895248274034, + "nauc_recall_at_1000_diff1": -0.5636061826821229, + "nauc_recall_at_1000_max": 1.0, + "nauc_recall_at_1000_std": 1.0, + "nauc_recall_at_100_diff1": -1.0972131138256949, + "nauc_recall_at_100_max": 0.678845778980395, + "nauc_recall_at_100_std": -0.005468504958684773, + "nauc_recall_at_10_diff1": -1.0232817354629227, + "nauc_recall_at_10_max": 0.5389931775812146, + "nauc_recall_at_10_std": -0.28544906131199166, + "nauc_recall_at_1_diff1": 0.7903694326100982, + "nauc_recall_at_1_max": 0.05856215046883633, + "nauc_recall_at_1_std": -0.12094234457399146, + "nauc_recall_at_20_diff1": -1.036891106974818, + "nauc_recall_at_20_max": 0.5361309760012383, + "nauc_recall_at_20_std": -0.2835267458980096, + "nauc_recall_at_3_diff1": -1.033270818345791, + "nauc_recall_at_3_max": 0.43638107488012995, + "nauc_recall_at_3_std": -0.36871033246815943, + "nauc_recall_at_5_diff1": -1.0189881929348006, + "nauc_recall_at_5_max": 0.47559438109485574, + "nauc_recall_at_5_std": -0.35994895248274084, + "ndcg_at_1": 0.0523, + "ndcg_at_10": 0.57563, + "ndcg_at_100": 0.5872, + "ndcg_at_1000": 0.58814, + "ndcg_at_20": 0.58126, + "ndcg_at_3": 0.53858, + "ndcg_at_5": 0.56133, + "precision_at_1": 0.0523, + "precision_at_10": 0.09397, + "precision_at_100": 0.00993, + "precision_at_1000": 0.001, + "precision_at_20": 0.04809, + "precision_at_3": 0.28035, + "precision_at_5": 0.17918, + "recall_at_1": 0.0523, + "recall_at_10": 0.93967, + "recall_at_100": 0.99265, + "recall_at_1000": 0.99983, + "recall_at_20": 0.96172, + "recall_at_3": 0.84105, + "recall_at_5": 0.89592 + } + ] + }, + "task_name": "SyntheticText2SQL" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/XPQARetrieval.json b/results/voyageai__voyage-3/no_revision_available/XPQARetrieval.json new file mode 100644 index 000000000..3c14cccbc --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/XPQARetrieval.json @@ -0,0 +1,159 @@ +{ + "dataset_revision": "c99d599f0a6ab9b85b065da6f9d94f9cf731679f", + "evaluation_time": 7.127863168716431, + "kg_co2_emissions": null, + "mteb_version": "1.14.12", + "scores": { + "test": [ + { + "hf_subset": "fra-fra", + "languages": [ + "fra-Latn", + "fra-Latn" + ], + "main_score": 0.76081, + "map_at_1": 0.45679, + "map_at_10": 0.70251, + "map_at_100": 0.71447, + "map_at_1000": 0.71475, + "map_at_20": 0.71074, + "map_at_3": 0.6335, + "map_at_5": 0.67965, + "mrr_at_1": 0.7116154873164219, + "mrr_at_10": 0.7756755038463978, + "mrr_at_100": 0.7790842700555463, + "mrr_at_1000": 0.7791548546003378, + "mrr_at_20": 0.7780621718599389, + "mrr_at_3": 0.7574543836226079, + "mrr_at_5": 0.7684023141967066, + "nauc_map_at_1000_diff1": 0.47058617130778757, + "nauc_map_at_1000_max": 0.492704777467116, + "nauc_map_at_1000_std": 0.005317319867245561, + "nauc_map_at_100_diff1": 0.4703406134367556, + "nauc_map_at_100_max": 0.49277288015993853, + "nauc_map_at_100_std": 0.005861810920222426, + "nauc_map_at_10_diff1": 0.4676676525689699, + "nauc_map_at_10_max": 0.48145953556591503, + "nauc_map_at_10_std": -0.0037736256358269532, + "nauc_map_at_1_diff1": 0.5793182034683065, + "nauc_map_at_1_max": 0.1849831597583719, + "nauc_map_at_1_std": -0.10841965678693878, + "nauc_map_at_20_diff1": 0.4701222492548738, + "nauc_map_at_20_max": 0.490703198674493, + "nauc_map_at_20_std": 0.003789979555559355, + "nauc_map_at_3_diff1": 0.4931836601250146, + "nauc_map_at_3_max": 0.37647265072972796, + "nauc_map_at_3_std": -0.04797726286365257, + "nauc_map_at_5_diff1": 0.46975952545664, + "nauc_map_at_5_max": 0.45399813017718854, + "nauc_map_at_5_std": -0.02331976758283398, + "nauc_mrr_at_1000_diff1": 0.5577595730466418, + "nauc_mrr_at_1000_max": 0.6380122369465924, + "nauc_mrr_at_1000_std": 0.05195271466105717, + "nauc_mrr_at_100_diff1": 0.5577343591733912, + "nauc_mrr_at_100_max": 0.6381029300024809, + "nauc_mrr_at_100_std": 0.052176883005224335, + "nauc_mrr_at_10_diff1": 0.5558998885968853, + "nauc_mrr_at_10_max": 0.639369584398442, + "nauc_mrr_at_10_std": 0.049495746877406745, + "nauc_mrr_at_1_diff1": 0.5719281114377524, + "nauc_mrr_at_1_max": 0.6116767821679298, + "nauc_mrr_at_1_std": 0.043475406998717506, + "nauc_mrr_at_20_diff1": 0.5572128635018695, + "nauc_mrr_at_20_max": 0.6385176266674133, + "nauc_mrr_at_20_std": 0.05291242092252573, + "nauc_mrr_at_3_diff1": 0.5635769132823579, + "nauc_mrr_at_3_max": 0.6371401083186427, + "nauc_mrr_at_3_std": 0.04147775317142181, + "nauc_mrr_at_5_diff1": 0.5571659557705373, + "nauc_mrr_at_5_max": 0.639785037619975, + "nauc_mrr_at_5_std": 0.04497114876904889, + "nauc_ndcg_at_1000_diff1": 0.48975812564175386, + "nauc_ndcg_at_1000_max": 0.5545170663641815, + "nauc_ndcg_at_1000_std": 0.03309116648549778, + "nauc_ndcg_at_100_diff1": 0.4855388939092838, + "nauc_ndcg_at_100_max": 0.5563342026856971, + "nauc_ndcg_at_100_std": 0.04301103437742535, + "nauc_ndcg_at_10_diff1": 0.4724635542742135, + "nauc_ndcg_at_10_max": 0.5326413281211997, + "nauc_ndcg_at_10_std": 0.014369532399353884, + "nauc_ndcg_at_1_diff1": 0.5719281114377524, + "nauc_ndcg_at_1_max": 0.6116767821679298, + "nauc_ndcg_at_1_std": 0.043475406998717506, + "nauc_ndcg_at_20_diff1": 0.48353575077647043, + "nauc_ndcg_at_20_max": 0.5514619236401542, + "nauc_ndcg_at_20_std": 0.036647697497961595, + "nauc_ndcg_at_3_diff1": 0.48598790126832997, + "nauc_ndcg_at_3_max": 0.5147762870702012, + "nauc_ndcg_at_3_std": 0.0017125030028531773, + "nauc_ndcg_at_5_diff1": 0.4757973977382607, + "nauc_ndcg_at_5_max": 0.5014642323150713, + "nauc_ndcg_at_5_std": -0.014091168544251945, + "nauc_precision_at_1000_diff1": -0.2705285630472523, + "nauc_precision_at_1000_max": 0.2533410635307587, + "nauc_precision_at_1000_std": 0.20436104448055145, + "nauc_precision_at_100_diff1": -0.266113477733609, + "nauc_precision_at_100_max": 0.27335149873889303, + "nauc_precision_at_100_std": 0.2228700168574349, + "nauc_precision_at_10_diff1": -0.19385959205653966, + "nauc_precision_at_10_max": 0.35563604130933685, + "nauc_precision_at_10_std": 0.19452957434700704, + "nauc_precision_at_1_diff1": 0.5719281114377524, + "nauc_precision_at_1_max": 0.6116767821679298, + "nauc_precision_at_1_std": 0.043475406998717506, + "nauc_precision_at_20_diff1": -0.22332802249594605, + "nauc_precision_at_20_max": 0.328487724415424, + "nauc_precision_at_20_std": 0.2151640290976394, + "nauc_precision_at_3_diff1": -0.0401299697643658, + "nauc_precision_at_3_max": 0.4309985357520983, + "nauc_precision_at_3_std": 0.14348065905522647, + "nauc_precision_at_5_diff1": -0.12444741619567504, + "nauc_precision_at_5_max": 0.3984111547364592, + "nauc_precision_at_5_std": 0.15860085623080922, + "nauc_recall_at_1000_diff1": NaN, + "nauc_recall_at_1000_max": NaN, + "nauc_recall_at_1000_std": NaN, + "nauc_recall_at_100_diff1": 0.2672715705997225, + "nauc_recall_at_100_max": 0.6356825969898798, + "nauc_recall_at_100_std": 0.523107558595024, + "nauc_recall_at_10_diff1": 0.3478374087279442, + "nauc_recall_at_10_max": 0.46876332030258283, + "nauc_recall_at_10_std": -0.0006184357651029486, + "nauc_recall_at_1_diff1": 0.5793182034683065, + "nauc_recall_at_1_max": 0.1849831597583719, + "nauc_recall_at_1_std": -0.10841965678693878, + "nauc_recall_at_20_diff1": 0.3817250779730652, + "nauc_recall_at_20_max": 0.5403107390729486, + "nauc_recall_at_20_std": 0.13490401980707492, + "nauc_recall_at_3_diff1": 0.45825955085558445, + "nauc_recall_at_3_max": 0.3392888734141155, + "nauc_recall_at_3_std": -0.0699141935008427, + "nauc_recall_at_5_diff1": 0.3959789294824256, + "nauc_recall_at_5_max": 0.4289971634400706, + "nauc_recall_at_5_std": -0.05667588613016716, + "ndcg_at_1": 0.71162, + "ndcg_at_10": 0.76081, + "ndcg_at_100": 0.79522, + "ndcg_at_1000": 0.79891, + "ndcg_at_20": 0.77968, + "ndcg_at_3": 0.70856, + "ndcg_at_5": 0.72541, + "precision_at_1": 0.71162, + "precision_at_10": 0.17423, + "precision_at_100": 0.02024, + "precision_at_1000": 0.00207, + "precision_at_20": 0.09393, + "precision_at_3": 0.43124, + "precision_at_5": 0.30708, + "recall_at_1": 0.45679, + "recall_at_10": 0.8506, + "recall_at_100": 0.97713, + "recall_at_1000": 1.0, + "recall_at_20": 0.91119, + "recall_at_3": 0.67888, + "recall_at_5": 0.76533 + } + ] + }, + "task_name": "XPQARetrieval" +} \ No newline at end of file diff --git a/results/voyageai__voyage-3/no_revision_available/model_meta.json b/results/voyageai__voyage-3/no_revision_available/model_meta.json new file mode 100644 index 000000000..ca8563ca5 --- /dev/null +++ b/results/voyageai__voyage-3/no_revision_available/model_meta.json @@ -0,0 +1 @@ +{"name": "voyageai__voyage-3", "revision": "no_revision_available", "release_date": "2024-09-22", "languages": null, "n_parameters": null, "memory_usage": null, "max_tokens": 32000, "embed_dim": 1024, "license": null, "open_source": false, "similarity_fn_name": null, "framework": [], "loader": "VoyageWrapper"} \ No newline at end of file