init

.github/workflows/main.yml

@@ -0,0 +1,19 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://mteb:[email protected]/datasets/mteb/results main

CONTRIBUTING

@@ -0,0 +1,5 @@
+TODO
+
+Users must be sure no files are over 10MB and so must compress them with gzip (e.g. compress_great_than_10mb.py)
+
+This should be a pre-commit hook checking.

README.md

@@ -0,0 +1,5 @@
+---
+benchmark: mteb
+type: evaluation
+submission_name: MTEB
+---

compress_greater_than_10mb.py

@@ -0,0 +1,10 @@
+import os
+import glob
+import sys
+
+for file in glob.glob("results/*/*/*.json"):
+    # if the file is greater than 9 MB, compress it with gzip
+    if os.path.getsize(file) >= 9 * 1024 * 1024:
+        # gzip it but name it as a json file
+        os.system(f"gzip {file} -c > {file}.gz")
+        print(f"Compressed {file} with gzip.")

convert_gzip_to_json.sh

@@ -0,0 +1,19 @@
+
+files=(
+    results/GritLM-7B/13f00a0e36500c80ce12870ea513846a066004af/FloresBitextMining.json.gz
+    results/LaBSE/e34fab64a3011d2176c99545a93d5cbddc9a91b7/FloresBitextMining.json.gz
+    results/all-MiniLM-L12-v2/a05860a77cef7b37e0048a7864658139bc18a854/FloresBitextMining.json.gz
+    results/all-MiniLM-L6-v2/8b3219a92973c328a8e22fadcfa821b5dc75636a/FloresBitextMining.json.gz
+    results/all-mpnet-base-v2/84f2bcc00d77236f9e89c8a360a00fb1139bf47d/FloresBitextMining.json.gz
+    results/all-mpnet-base-v2/no_revision_available/FloresBitextMining.json.gz
+    results/multilingual-e5-base/d13f1b27baf31030b7fd040960d60d909913633f/FloresBitextMining.json.gz
+    results/multilingual-e5-large-instruct/baa7be480a7de1539afce709c8f13f833a510e0a/FloresBitextMining.json.gz
+    results/multilingual-e5-large/4dc6d853a804b9c8886ede6dda8a073b7dc08a81/FloresBitextMining.json.gz
+    results/multilingual-e5-small/e4ce9877abf3edfe10b0d82785e83bdcb973e22e/FloresBitextMining.json.gz
+)
+
+for file in "${files[@]}"; do
+    echo "Converting $file"
+    # rename it as .json
+    mv $file ${file%.gz}
+done