Change boolq rc back to acc

allenai · Nov 19, 2024 · ee99d57 · ee99d57
1 parent 964062e
commit ee99d57
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/olmo/eval/downstream.py b/olmo/eval/downstream.py
@@ -1966,8 +1966,8 @@ def doc_to_label(self, doc) -> int:
     ),
     "boolq_train_rc_5shot": (
         OEEvalTask,
-        {"dataset_path": "boolq", "dataset_name": "train_rc_5shot", "metric_type": "len_norm"},
-    ),  # this used to be acc
+        {"dataset_path": "boolq", "dataset_name": "train_rc_5shot", "metric_type": "acc"},
+    ),  # kept acc here, since len_norm can bias towards "yes"
     "boolq_train_rc_5shot_bpb": (
         OEEvalTask,
         {"dataset_path": "boolq", "dataset_name": "train_rc_5shot", "metric_type": "bpb"},
@@ -1982,7 +1982,7 @@ def doc_to_label(self, doc) -> int:
     ),
     "boolq_val_rc_5shot": (
         OEEvalTask,
-        {"dataset_path": "boolq", "dataset_name": "val_rc_5shot", "metric_type": "len_norm"},
+        {"dataset_path": "boolq", "dataset_name": "val_rc_5shot", "metric_type": "acc"},
     ),
     "boolq_val_rc_5shot_bpb": (
         OEEvalTask,