Skip to content

Commit

Permalink
Add results of voyage-3 and voyage-3-lite (#35)
Browse files Browse the repository at this point in the history 
* Add results of voyage-3 and voyage-3-lite

* Added model_meta.json

* Fixed model names in model_meta.json

* Fixed MIRACL version

* Attempted to fix the miracl format again

* Attempted to fix MIRACL data format
  • Loading branch information
@Liuhong99
Liuhong99 authored Oct 6, 2024
1 parent 324c458 commit e7d51d9
Show file tree
Hide file tree
Showing 50 changed files with 12,859 additions and 0 deletions.
50 changes: 50 additions & 0 deletions paths.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13367,6 +13367,56 @@
"results/voyageai__voyage-multilingual-2/no_revision_available/XPQARetrieval.json",
"results/voyageai__voyage-multilingual-2/no_revision_available/STSBenchmarkMultilingualSTS.json"
],
"voyageai__voyage-3": [
"results/voyageai__voyage-3/no_revision_available/AILACasedocs.json",
"results/voyageai__voyage-3/no_revision_available/AILAStatutes.json",
"results/voyageai__voyage-3/no_revision_available/AlloprofRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/AppsRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/BSARDRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/CodeFeedbackMT.json",
"results/voyageai__voyage-3/no_revision_available/CodeFeedbackST.json",
"results/voyageai__voyage-3/no_revision_available/CodeSearchNetCCRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/CodeSearchNetRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/CodeTransOceanContest.json",
"results/voyageai__voyage-3/no_revision_available/CodeTransOceanDL.json",
"results/voyageai__voyage-3/no_revision_available/CosQA.json",
"results/voyageai__voyage-3/no_revision_available/GerDaLIRSmall.json",
"results/voyageai__voyage-3/no_revision_available/LEMBNarrativeQARetrieval.json",
"results/voyageai__voyage-3/no_revision_available/LEMBNeedleRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/LEMBPasskeyRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/LEMBQMSumRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/LEMBSummScreenFDRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/LEMBWikimQARetrieval.json",
"results/voyageai__voyage-3/no_revision_available/LeCaRDv2.json",
"results/voyageai__voyage-3/no_revision_available/LegalBenchConsumerContractsQA.json",
"results/voyageai__voyage-3/no_revision_available/LegalBenchCorporateLobbying.json",
"results/voyageai__voyage-3/no_revision_available/LegalQuAD.json",
"results/voyageai__voyage-3/no_revision_available/LegalSummarization.json",
"results/voyageai__voyage-3/no_revision_available/MIRACLRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/MintakaRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/RiaNewsRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/RuBQRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/StackOverflowQA.json",
"results/voyageai__voyage-3/no_revision_available/SyntecRetrieval.json",
"results/voyageai__voyage-3/no_revision_available/SyntheticText2SQL.json",
"results/voyageai__voyage-3/no_revision_available/XPQARetrieval.json"
],
"voyageai__voyage-3-lite": [
"results/voyageai__voyage-3-lite/no_revision_available/AILACasedocs.json",
"results/voyageai__voyage-3-lite/no_revision_available/AILAStatutes.json",
"results/voyageai__voyage-3-lite/no_revision_available/GerDaLIRSmall.json",
"results/voyageai__voyage-3-lite/no_revision_available/LEMBNarrativeQARetrieval.json",
"results/voyageai__voyage-3-lite/no_revision_available/LEMBNeedleRetrieval.json",
"results/voyageai__voyage-3-lite/no_revision_available/LEMBPasskeyRetrieval.json",
"results/voyageai__voyage-3-lite/no_revision_available/LEMBQMSumRetrieval.json",
"results/voyageai__voyage-3-lite/no_revision_available/LEMBSummScreenFDRetrieval.json",
"results/voyageai__voyage-3-lite/no_revision_available/LEMBWikimQARetrieval.json",
"results/voyageai__voyage-3-lite/no_revision_available/LeCaRDv2.json",
"results/voyageai__voyage-3-lite/no_revision_available/LegalBenchConsumerContractsQA.json",
"results/voyageai__voyage-3-lite/no_revision_available/LegalBenchCorporateLobbying.json",
"results/voyageai__voyage-3-lite/no_revision_available/LegalQuAD.json",
"results/voyageai__voyage-3-lite/no_revision_available/LegalSummarization.json"
],
"Muennighoff__SGPT-125M-weightedmean-msmarco-specb-bitfit": [
"results/Muennighoff__SGPT-125M-weightedmean-msmarco-specb-bitfit/no_revision_available/MassiveIntentClassification.json",
"results/Muennighoff__SGPT-125M-weightedmean-msmarco-specb-bitfit/no_revision_available/BiorxivClusteringP2P.json",
Expand Down
2 changes: 2 additions & 0 deletions results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,8 @@
"voyageai__voyage-lite-01-instruct",
"voyageai__voyage-lite-02-instruct",
"voyageai__voyage-multilingual-2",
"voyageai__voyage-3",
"voyageai__voyage-3-lite",
"vprelovac__universal-sentence-encoder-multilingual-3",
"vprelovac__universal-sentence-encoder-multilingual-large-3",
]
Expand Down
158 changes: 158 additions & 0 deletions results/voyageai__voyage-3-lite/no_revision_available/AILACasedocs.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "4106e6bcc72e0698d714ea8b101355e3e238431a",
"evaluation_time": 8.321073293685913,
"kg_co2_emissions": null,
"mteb_version": "1.14.12",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.38154,
"map_at_1": 0.11774,
"map_at_10": 0.2824,
"map_at_100": 0.3326,
"map_at_1000": 0.33454,
"map_at_20": 0.30578,
"map_at_3": 0.21326,
"map_at_5": 0.24743,
"mrr_at_1": 0.36,
"mrr_at_10": 0.45733333333333326,
"mrr_at_100": 0.4695673020668209,
"mrr_at_1000": 0.4695673020668209,
"mrr_at_20": 0.4637348484848484,
"mrr_at_3": 0.4233333333333333,
"mrr_at_5": 0.44233333333333325,
"nauc_map_at_1000_diff1": 0.22680444481492215,
"nauc_map_at_1000_max": 0.058264921235812575,
"nauc_map_at_1000_std": -0.3647929169273832,
"nauc_map_at_100_diff1": 0.22312948994886786,
"nauc_map_at_100_max": 0.05478784161725515,
"nauc_map_at_100_std": -0.36448202027822085,
"nauc_map_at_10_diff1": 0.2215987871258074,
"nauc_map_at_10_max": 0.0045721676125434155,
"nauc_map_at_10_std": -0.39561512229320683,
"nauc_map_at_1_diff1": 0.33826120576351165,
"nauc_map_at_1_max": 0.06003223626014147,
"nauc_map_at_1_std": -0.3849608871379809,
"nauc_map_at_20_diff1": 0.2121001502428955,
"nauc_map_at_20_max": 0.02000717925582694,
"nauc_map_at_20_std": -0.38079097083669483,
"nauc_map_at_3_diff1": 0.3108232789388227,
"nauc_map_at_3_max": 0.07251751092473184,
"nauc_map_at_3_std": -0.3818344258445587,
"nauc_map_at_5_diff1": 0.26047649421585917,
"nauc_map_at_5_max": 0.02893656086180195,
"nauc_map_at_5_std": -0.3803407350043284,
"nauc_mrr_at_1000_diff1": 0.13590981062873708,
"nauc_mrr_at_1000_max": 0.03794546498209027,
"nauc_mrr_at_1000_std": -0.3323213254706094,
"nauc_mrr_at_100_diff1": 0.13590981062873708,
"nauc_mrr_at_100_max": 0.03794546498209027,
"nauc_mrr_at_100_std": -0.3323213254706094,
"nauc_mrr_at_10_diff1": 0.12946164961838497,
"nauc_mrr_at_10_max": 0.04253237584217536,
"nauc_mrr_at_10_std": -0.3316373685619662,
"nauc_mrr_at_1_diff1": 0.13577914239227423,
"nauc_mrr_at_1_max": -0.03183225738728817,
"nauc_mrr_at_1_std": -0.359444706870309,
"nauc_mrr_at_20_diff1": 0.1351769803558538,
"nauc_mrr_at_20_max": 0.035118723107941084,
"nauc_mrr_at_20_std": -0.3312427934370972,
"nauc_mrr_at_3_diff1": 0.1655314639498021,
"nauc_mrr_at_3_max": 0.04594602913564839,
"nauc_mrr_at_3_std": -0.3438103236756822,
"nauc_mrr_at_5_diff1": 0.14054834158345927,
"nauc_mrr_at_5_max": 0.035727431560419354,
"nauc_mrr_at_5_std": -0.34653712153173016,
"nauc_ndcg_at_1000_diff1": 0.19399576593643414,
"nauc_ndcg_at_1000_max": 0.07372089639836343,
"nauc_ndcg_at_1000_std": -0.3381474164640091,
"nauc_ndcg_at_100_diff1": 0.17564092003061438,
"nauc_ndcg_at_100_max": 0.07343870665529059,
"nauc_ndcg_at_100_std": -0.3216818651714621,
"nauc_ndcg_at_10_diff1": 0.18541150098566386,
"nauc_ndcg_at_10_max": 0.03683917481009019,
"nauc_ndcg_at_10_std": -0.3775098629847817,
"nauc_ndcg_at_1_diff1": 0.13577914239227423,
"nauc_ndcg_at_1_max": -0.03183225738728817,
"nauc_ndcg_at_1_std": -0.359444706870309,
"nauc_ndcg_at_20_diff1": 0.1489261374086848,
"nauc_ndcg_at_20_max": 0.013220882910496612,
"nauc_ndcg_at_20_std": -0.3433646168711587,
"nauc_ndcg_at_3_diff1": 0.29960048545744683,
"nauc_ndcg_at_3_max": 0.1100822360591833,
"nauc_ndcg_at_3_std": -0.36610259960246705,
"nauc_ndcg_at_5_diff1": 0.2432713686658448,
"nauc_ndcg_at_5_max": 0.10419955769718153,
"nauc_ndcg_at_5_std": -0.35193778838882284,
"nauc_precision_at_1000_diff1": 0.1569166013898087,
"nauc_precision_at_1000_max": 0.3193332683366335,
"nauc_precision_at_1000_std": 0.05502755075625906,
"nauc_precision_at_100_diff1": 0.11406300006253677,
"nauc_precision_at_100_max": 0.31741716665256575,
"nauc_precision_at_100_std": 0.07248354736776305,
"nauc_precision_at_10_diff1": 0.04641435412219487,
"nauc_precision_at_10_max": 0.053856804938556616,
"nauc_precision_at_10_std": -0.23232417931114027,
"nauc_precision_at_1_diff1": 0.13577914239227423,
"nauc_precision_at_1_max": -0.03183225738728817,
"nauc_precision_at_1_std": -0.359444706870309,
"nauc_precision_at_20_diff1": 0.05806584045364722,
"nauc_precision_at_20_max": 0.18525645960023593,
"nauc_precision_at_20_std": -0.04514920586046764,
"nauc_precision_at_3_diff1": 0.24310475917620758,
"nauc_precision_at_3_max": 0.06401031393593481,
"nauc_precision_at_3_std": -0.33590814426287297,
"nauc_precision_at_5_diff1": 0.1374756938306522,
"nauc_precision_at_5_max": 0.06846385009722461,
"nauc_precision_at_5_std": -0.2698957044369809,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": -0.23264557740078837,
"nauc_recall_at_100_max": 0.159802171372661,
"nauc_recall_at_100_std": 0.2920852872225342,
"nauc_recall_at_10_diff1": 0.04261035679343841,
"nauc_recall_at_10_max": -0.03225531214913665,
"nauc_recall_at_10_std": -0.29384996013645287,
"nauc_recall_at_1_diff1": 0.33826120576351165,
"nauc_recall_at_1_max": 0.06003223626014147,
"nauc_recall_at_1_std": -0.3849608871379809,
"nauc_recall_at_20_diff1": -0.0412390471587858,
"nauc_recall_at_20_max": -0.07771184412953082,
"nauc_recall_at_20_std": -0.19523810164173544,
"nauc_recall_at_3_diff1": 0.29707784088826306,
"nauc_recall_at_3_max": 0.11162099294883046,
"nauc_recall_at_3_std": -0.35102019917685096,
"nauc_recall_at_5_diff1": 0.1943414621849361,
"nauc_recall_at_5_max": 0.03137833596530802,
"nauc_recall_at_5_std": -0.35136038822591403,
"ndcg_at_1": 0.36,
"ndcg_at_10": 0.38154,
"ndcg_at_100": 0.52174,
"ndcg_at_1000": 0.53267,
"ndcg_at_20": 0.4218,
"ndcg_at_3": 0.34325,
"ndcg_at_5": 0.34495,
"precision_at_1": 0.36,
"precision_at_10": 0.164,
"precision_at_100": 0.0362,
"precision_at_1000": 0.0039,
"precision_at_20": 0.111,
"precision_at_3": 0.28667,
"precision_at_5": 0.232,
"recall_at_1": 0.11774,
"recall_at_10": 0.44887,
"recall_at_100": 0.95603,
"recall_at_1000": 1.0,
"recall_at_20": 0.56508,
"recall_at_3": 0.24251,
"recall_at_5": 0.31412
}
]
},
"task_name": "AILACasedocs"
}
158 changes: 158 additions & 0 deletions results/voyageai__voyage-3-lite/no_revision_available/AILAStatutes.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e",
"evaluation_time": 3.139857292175293,
"kg_co2_emissions": null,
"mteb_version": "1.14.12",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.35033,
"map_at_1": 0.08,
"map_at_10": 0.21573,
"map_at_100": 0.293,
"map_at_1000": 0.293,
"map_at_20": 0.25577,
"map_at_3": 0.14072,
"map_at_5": 0.17969,
"mrr_at_1": 0.34,
"mrr_at_10": 0.5111031746031744,
"mrr_at_100": 0.5151237373737372,
"mrr_at_1000": 0.5151237373737372,
"mrr_at_20": 0.5141713564213563,
"mrr_at_3": 0.43999999999999995,
"mrr_at_5": 0.4919999999999998,
"nauc_map_at_1000_diff1": 0.2627428049451818,
"nauc_map_at_1000_max": 0.37195611611594037,
"nauc_map_at_1000_std": 0.1695908199424899,
"nauc_map_at_100_diff1": 0.2627428049451818,
"nauc_map_at_100_max": 0.37195611611594037,
"nauc_map_at_100_std": 0.1695908199424899,
"nauc_map_at_10_diff1": 0.257026135451343,
"nauc_map_at_10_max": 0.358611488616898,
"nauc_map_at_10_std": 0.17561884128144586,
"nauc_map_at_1_diff1": 0.278328846552081,
"nauc_map_at_1_max": 0.43277593704700756,
"nauc_map_at_1_std": 0.30501138952164003,
"nauc_map_at_20_diff1": 0.2415186441540987,
"nauc_map_at_20_max": 0.3631129644969627,
"nauc_map_at_20_std": 0.180440504561769,
"nauc_map_at_3_diff1": 0.2914087617281748,
"nauc_map_at_3_max": 0.37794665974262803,
"nauc_map_at_3_std": 0.16516069465199582,
"nauc_map_at_5_diff1": 0.2346491426208135,
"nauc_map_at_5_max": 0.3591654170954656,
"nauc_map_at_5_std": 0.1769291260967869,
"nauc_mrr_at_1000_diff1": 0.3296532710194783,
"nauc_mrr_at_1000_max": 0.4148119538338282,
"nauc_mrr_at_1000_std": 0.29561611808491245,
"nauc_mrr_at_100_diff1": 0.3296532710194783,
"nauc_mrr_at_100_max": 0.4148119538338282,
"nauc_mrr_at_100_std": 0.29561611808491245,
"nauc_mrr_at_10_diff1": 0.3226708021374957,
"nauc_mrr_at_10_max": 0.4105946066597358,
"nauc_mrr_at_10_std": 0.2948483005668789,
"nauc_mrr_at_1_diff1": 0.3545138980895093,
"nauc_mrr_at_1_max": 0.45496701025464353,
"nauc_mrr_at_1_std": 0.3394101592517023,
"nauc_mrr_at_20_diff1": 0.32897595485222514,
"nauc_mrr_at_20_max": 0.4128928483014366,
"nauc_mrr_at_20_std": 0.29408727186886474,
"nauc_mrr_at_3_diff1": 0.36847244552234065,
"nauc_mrr_at_3_max": 0.41595134933827443,
"nauc_mrr_at_3_std": 0.23793952275844535,
"nauc_mrr_at_5_diff1": 0.2976288373079473,
"nauc_mrr_at_5_max": 0.4014494884923278,
"nauc_mrr_at_5_std": 0.278804694706051,
"nauc_ndcg_at_1000_diff1": 0.2875011003018454,
"nauc_ndcg_at_1000_max": 0.3778453138654806,
"nauc_ndcg_at_1000_std": 0.195237859804941,
"nauc_ndcg_at_100_diff1": 0.2875011003018454,
"nauc_ndcg_at_100_max": 0.3778453138654806,
"nauc_ndcg_at_100_std": 0.195237859804941,
"nauc_ndcg_at_10_diff1": 0.2616213255109694,
"nauc_ndcg_at_10_max": 0.351841818006536,
"nauc_ndcg_at_10_std": 0.20553076558837527,
"nauc_ndcg_at_1_diff1": 0.3545138980895093,
"nauc_ndcg_at_1_max": 0.45496701025464353,
"nauc_ndcg_at_1_std": 0.3394101592517023,
"nauc_ndcg_at_20_diff1": 0.23366274364875173,
"nauc_ndcg_at_20_max": 0.3366167411090695,
"nauc_ndcg_at_20_std": 0.17776346311216973,
"nauc_ndcg_at_3_diff1": 0.3281798973074511,
"nauc_ndcg_at_3_max": 0.3641306132599758,
"nauc_ndcg_at_3_std": 0.1546357549276064,
"nauc_ndcg_at_5_diff1": 0.21731337842351942,
"nauc_ndcg_at_5_max": 0.36072544427054365,
"nauc_ndcg_at_5_std": 0.19313711550233276,
"nauc_precision_at_1000_diff1": 0.051517535354273766,
"nauc_precision_at_1000_max": -0.086569949324014,
"nauc_precision_at_1000_std": -0.1009311940871905,
"nauc_precision_at_100_diff1": 0.05151753535427775,
"nauc_precision_at_100_max": -0.08656994932401264,
"nauc_precision_at_100_std": -0.10093119408718829,
"nauc_precision_at_10_diff1": 0.18854668275574313,
"nauc_precision_at_10_max": 0.21657099680849856,
"nauc_precision_at_10_std": 0.12317165315259393,
"nauc_precision_at_1_diff1": 0.3545138980895093,
"nauc_precision_at_1_max": 0.45496701025464353,
"nauc_precision_at_1_std": 0.3394101592517023,
"nauc_precision_at_20_diff1": 0.08028870363117518,
"nauc_precision_at_20_max": 0.12511423189540946,
"nauc_precision_at_20_std": 0.04167366045618141,
"nauc_precision_at_3_diff1": 0.324571784490813,
"nauc_precision_at_3_max": 0.31826533790096534,
"nauc_precision_at_3_std": 0.043864839613827686,
"nauc_precision_at_5_diff1": 0.11475019825535299,
"nauc_precision_at_5_max": 0.26426116838488006,
"nauc_precision_at_5_std": 0.12357916997092283,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": NaN,
"nauc_recall_at_100_max": NaN,
"nauc_recall_at_100_std": NaN,
"nauc_recall_at_10_diff1": 0.17516056043860467,
"nauc_recall_at_10_max": 0.23446871464624489,
"nauc_recall_at_10_std": 0.1445740144460882,
"nauc_recall_at_1_diff1": 0.278328846552081,
"nauc_recall_at_1_max": 0.43277593704700756,
"nauc_recall_at_1_std": 0.30501138952164003,
"nauc_recall_at_20_diff1": 0.07020015280087248,
"nauc_recall_at_20_max": 0.16190028721661406,
"nauc_recall_at_20_std": 0.05956946107050535,
"nauc_recall_at_3_diff1": 0.2737050070501919,
"nauc_recall_at_3_max": 0.32529362331371153,
"nauc_recall_at_3_std": 0.05578741954903067,
"nauc_recall_at_5_diff1": 0.08950675622720845,
"nauc_recall_at_5_max": 0.26082033645135566,
"nauc_recall_at_5_std": 0.11142911147890128,
"ndcg_at_1": 0.34,
"ndcg_at_10": 0.35033,
"ndcg_at_100": 0.55089,
"ndcg_at_1000": 0.55089,
"ndcg_at_20": 0.43137,
"ndcg_at_3": 0.27508,
"ndcg_at_5": 0.28528,
"precision_at_1": 0.34,
"precision_at_10": 0.176,
"precision_at_100": 0.0434,
"precision_at_1000": 0.00434,
"precision_at_20": 0.133,
"precision_at_3": 0.25333,
"precision_at_5": 0.24,
"recall_at_1": 0.08,
"recall_at_10": 0.40733,
"recall_at_100": 1.0,
"recall_at_1000": 1.0,
"recall_at_20": 0.59933,
"recall_at_3": 0.18267,
"recall_at_5": 0.28633
}
]
},
"task_name": "AILAStatutes"
}
Loading

0 comments on commit e7d51d9

Please sign in to comment.