Skip to content

Commit

Permalink
Add results (#40)
Browse files Browse the repository at this point in the history 
* Add

* Add res

* fix: reduce sizes of files and added makefile command

* Add

* Add res

* formatted flores files

* Add

* Add res

* make preprush

* Add OAI T3Small MMTEB

* update ci dependencies to >3.9

* tmp

* reduce large file sizes

* Add overview table to readme

* Add missing TE3L results

* Make prepush

* Add res

* reduce size

* AddRes

* Add

* fix: run make pre-push

* add revision to openai text-embedding-large, add "openai/" to model name

* minor fix to model meta

* make sure pip is upgraded installing packages

---------

Co-authored-by: Kenneth Enevoldsen <[email protected]>
  • Loading branch information
@Muennighoff @KennethEnevoldsen
Muennighoff and KennethEnevoldsen authored Nov 13, 2024
1 parent 1616d1b commit db0aadb
Show file tree
Hide file tree
Showing 2,506 changed files with 2,886,592 additions and 367 deletions.
1 change: 0 additions & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ jobs:
matrix:
os: [ubuntu-latest] #, macos-latest, windows-latest]
python-version: ["3.9"]

steps:
- uses: actions/checkout@v3

Expand Down
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ tmp.py

# MacOS
.DS_Store

# UV
uv.lock
18 changes: 17 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,20 @@
benchmark: mteb
type: evaluation
submission_name: MTEB
---
---

This repository contain the results of the embedding benchmark evaluated using the package `mteb`.


| Reference | |
| ------------------- | ---------------------------------------------------------------------------------------- |
| 🦾 **[Leaderboard]** | An up to date leaderboard of embedding models |
| 📚 **[mteb]** | Guides and instructions on how to use `mteb`, including running, submitting scores, etc. |
| 🙋 **[Questions]** | Questions about the results |
| 🙋 **[Issues]** | Issues or bugs you have found |


[Leaderboard]: https://huggingface.co/spaces/mteb/leaderboard
[mteb]: https://github.com/embeddings-benchmark/mteb
[Questions]: https://github.com/embeddings-benchmark/mteb/discussions
[Issues]: https://github.com/embeddings-benchmark/mteb/issues
4 changes: 2 additions & 2 deletions makefile
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
install-for-tests:
@echo "--- Installing dependencies for tests ---"
# just use the dev dependencies from mteb to keep everything compatible
pip install "mteb[dev]>=1.13.0"
pip install pip --upgrade
pip install .

test:
@echo "--- Running tests ---"
Expand Down
7 changes: 7 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[project]
name = "results"
version = "0.1.0"
description = "The result repository for mteb"
readme = "README.md"
requires-python = ">=3.9"
dependencies = ["mteb[dev]>=1.13.0"]
158 changes: 158 additions & 0 deletions ...aba-NLP__gte-Qwen2-7B-instruct/e26182b2122f4435e8b3ebecbf363990f409b45b/AILAStatutes.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
{
"dataset_revision": "ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e",
"evaluation_time": 17.8603093624115,
"kg_co2_emissions": 0.0017055287025926752,
"mteb_version": "1.18.0",
"scores": {
"test": [
{
"hf_subset": "default",
"languages": [
"eng-Latn"
],
"main_score": 0.33765,
"map_at_1": 0.084,
"map_at_10": 0.21694,
"map_at_100": 0.28429,
"map_at_1000": 0.28429,
"map_at_20": 0.24574,
"map_at_3": 0.1425,
"map_at_5": 0.18367,
"mrr_at_1": 0.3,
"mrr_at_10": 0.454095238095238,
"mrr_at_100": 0.46212957304133767,
"mrr_at_1000": 0.46212957304133767,
"mrr_at_20": 0.4612962397080043,
"mrr_at_3": 0.39333333333333337,
"mrr_at_5": 0.42533333333333334,
"nauc_map_at_1000_diff1": 0.1257589273091899,
"nauc_map_at_1000_max": 0.4362811047451273,
"nauc_map_at_1000_std": 0.15526294395684537,
"nauc_map_at_100_diff1": 0.1257589273091899,
"nauc_map_at_100_max": 0.4362811047451273,
"nauc_map_at_100_std": 0.15526294395684537,
"nauc_map_at_10_diff1": 0.12251716861218423,
"nauc_map_at_10_max": 0.44750464684869273,
"nauc_map_at_10_std": 0.1665813067303774,
"nauc_map_at_1_diff1": 0.1603550346774055,
"nauc_map_at_1_max": 0.3557488770716093,
"nauc_map_at_1_std": -0.03334365911163875,
"nauc_map_at_20_diff1": 0.11168162103830682,
"nauc_map_at_20_max": 0.42834021224498536,
"nauc_map_at_20_std": 0.18809254528446484,
"nauc_map_at_3_diff1": 0.1820738040919183,
"nauc_map_at_3_max": 0.43576454065851916,
"nauc_map_at_3_std": 0.03987196225744017,
"nauc_map_at_5_diff1": 0.1516707675586077,
"nauc_map_at_5_max": 0.4343805548656787,
"nauc_map_at_5_std": 0.12289883063537198,
"nauc_mrr_at_1000_diff1": 0.21369110302271785,
"nauc_mrr_at_1000_max": 0.4482852535080846,
"nauc_mrr_at_1000_std": 0.04709212363192428,
"nauc_mrr_at_100_diff1": 0.21369110302271785,
"nauc_mrr_at_100_max": 0.4482852535080846,
"nauc_mrr_at_100_std": 0.04709212363192428,
"nauc_mrr_at_10_diff1": 0.20469773018048376,
"nauc_mrr_at_10_max": 0.4376008078285497,
"nauc_mrr_at_10_std": 0.05364942361920574,
"nauc_mrr_at_1_diff1": 0.2885015538440751,
"nauc_mrr_at_1_max": 0.4731252533441427,
"nauc_mrr_at_1_std": -0.03950817457100396,
"nauc_mrr_at_20_diff1": 0.2115713156894796,
"nauc_mrr_at_20_max": 0.4485740159869434,
"nauc_mrr_at_20_std": 0.04933466533602979,
"nauc_mrr_at_3_diff1": 0.21908223819854072,
"nauc_mrr_at_3_max": 0.46955286229090865,
"nauc_mrr_at_3_std": 0.027972999623501044,
"nauc_mrr_at_5_diff1": 0.18683833661325422,
"nauc_mrr_at_5_max": 0.44082372073744047,
"nauc_mrr_at_5_std": 0.07459968781624475,
"nauc_ndcg_at_1000_diff1": 0.14747261650165755,
"nauc_ndcg_at_1000_max": 0.439315617438903,
"nauc_ndcg_at_1000_std": 0.13888603634732072,
"nauc_ndcg_at_100_diff1": 0.14747261650165755,
"nauc_ndcg_at_100_max": 0.439315617438903,
"nauc_ndcg_at_100_std": 0.13888603634732072,
"nauc_ndcg_at_10_diff1": 0.10151157379963689,
"nauc_ndcg_at_10_max": 0.4287131093173576,
"nauc_ndcg_at_10_std": 0.21756532293248818,
"nauc_ndcg_at_1_diff1": 0.2885015538440751,
"nauc_ndcg_at_1_max": 0.4731252533441427,
"nauc_ndcg_at_1_std": -0.03950817457100396,
"nauc_ndcg_at_20_diff1": 0.08771431706990646,
"nauc_ndcg_at_20_max": 0.41146211509813463,
"nauc_ndcg_at_20_std": 0.25283813479379036,
"nauc_ndcg_at_3_diff1": 0.20179057017543273,
"nauc_ndcg_at_3_max": 0.4596305993876074,
"nauc_ndcg_at_3_std": 0.05068535832174184,
"nauc_ndcg_at_5_diff1": 0.1346805085198524,
"nauc_ndcg_at_5_max": 0.434665851425462,
"nauc_ndcg_at_5_std": 0.176648112448584,
"nauc_precision_at_1000_diff1": -0.001920930828195194,
"nauc_precision_at_1000_max": -0.2500503100931195,
"nauc_precision_at_1000_std": -0.25191635718336514,
"nauc_precision_at_100_diff1": -0.0019209308281937865,
"nauc_precision_at_100_max": -0.25005031009311424,
"nauc_precision_at_100_std": -0.2519163571833613,
"nauc_precision_at_10_diff1": -0.018183872856718023,
"nauc_precision_at_10_max": 0.264066818027717,
"nauc_precision_at_10_std": 0.2958423653417436,
"nauc_precision_at_1_diff1": 0.2885015538440751,
"nauc_precision_at_1_max": 0.4731252533441427,
"nauc_precision_at_1_std": -0.03950817457100396,
"nauc_precision_at_20_diff1": -0.06991681775526426,
"nauc_precision_at_20_max": 0.10627830064049434,
"nauc_precision_at_20_std": 0.29626727648757734,
"nauc_precision_at_3_diff1": 0.19630950310198872,
"nauc_precision_at_3_max": 0.4547747434336407,
"nauc_precision_at_3_std": 0.09030556038731391,
"nauc_precision_at_5_diff1": 0.06223329362070833,
"nauc_precision_at_5_max": 0.3396512509476874,
"nauc_precision_at_5_std": 0.25769522365428343,
"nauc_recall_at_1000_diff1": NaN,
"nauc_recall_at_1000_max": NaN,
"nauc_recall_at_1000_std": NaN,
"nauc_recall_at_100_diff1": NaN,
"nauc_recall_at_100_max": NaN,
"nauc_recall_at_100_std": NaN,
"nauc_recall_at_10_diff1": -0.018606080799264093,
"nauc_recall_at_10_max": 0.34265022294542885,
"nauc_recall_at_10_std": 0.38239558029948173,
"nauc_recall_at_1_diff1": 0.1603550346774055,
"nauc_recall_at_1_max": 0.3557488770716093,
"nauc_recall_at_1_std": -0.03334365911163875,
"nauc_recall_at_20_diff1": -0.07765209201197695,
"nauc_recall_at_20_max": 0.20771577314708548,
"nauc_recall_at_20_std": 0.430869184234819,
"nauc_recall_at_3_diff1": 0.15971238444535218,
"nauc_recall_at_3_max": 0.44263477633603043,
"nauc_recall_at_3_std": 0.10476761601816997,
"nauc_recall_at_5_diff1": 0.061713450179887205,
"nauc_recall_at_5_max": 0.374497951016796,
"nauc_recall_at_5_std": 0.2932912634434461,
"ndcg_at_1": 0.3,
"ndcg_at_10": 0.33765,
"ndcg_at_100": 0.53487,
"ndcg_at_1000": 0.53487,
"ndcg_at_20": 0.40329,
"ndcg_at_3": 0.26148,
"ndcg_at_5": 0.2707,
"precision_at_1": 0.3,
"precision_at_10": 0.17,
"precision_at_100": 0.0434,
"precision_at_1000": 0.00434,
"precision_at_20": 0.122,
"precision_at_3": 0.24,
"precision_at_5": 0.224,
"recall_at_1": 0.084,
"recall_at_10": 0.39633,
"recall_at_100": 1.0,
"recall_at_1000": 1.0,
"recall_at_20": 0.558,
"recall_at_3": 0.177,
"recall_at_5": 0.26767
}
]
},
"task_name": "AILAStatutes"
}
Loading

0 comments on commit db0aadb

Please sign in to comment.