From ee99d57ab153be8a78cc90255df5224ef7bf6b4e Mon Sep 17 00:00:00 2001 From: Jiacheng Liu Date: Tue, 19 Nov 2024 00:07:45 +0000 Subject: [PATCH] Change boolq rc back to acc --- olmo/eval/downstream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/olmo/eval/downstream.py b/olmo/eval/downstream.py index fddd2c3f8..80342cb57 100644 --- a/olmo/eval/downstream.py +++ b/olmo/eval/downstream.py @@ -1966,8 +1966,8 @@ def doc_to_label(self, doc) -> int: ), "boolq_train_rc_5shot": ( OEEvalTask, - {"dataset_path": "boolq", "dataset_name": "train_rc_5shot", "metric_type": "len_norm"}, - ), # this used to be acc + {"dataset_path": "boolq", "dataset_name": "train_rc_5shot", "metric_type": "acc"}, + ), # kept acc here, since len_norm can bias towards "yes" "boolq_train_rc_5shot_bpb": ( OEEvalTask, {"dataset_path": "boolq", "dataset_name": "train_rc_5shot", "metric_type": "bpb"}, @@ -1982,7 +1982,7 @@ def doc_to_label(self, doc) -> int: ), "boolq_val_rc_5shot": ( OEEvalTask, - {"dataset_path": "boolq", "dataset_name": "val_rc_5shot", "metric_type": "len_norm"}, + {"dataset_path": "boolq", "dataset_name": "val_rc_5shot", "metric_type": "acc"}, ), "boolq_val_rc_5shot_bpb": ( OEEvalTask,