Eval peteish7 and peteish13 with new eval

configs/peteish13-weka.yaml

@@ -228,6 +228,126 @@ evaluators:
   - label: arc_easy_ppl
     type: downstream
 
+  - label: arc_challenge_val_rc_5shot
+    type: downstream
+
+  - label: arc_challenge_val_mc_5shot
+    type: downstream
+
+  - label: arc_challenge_test_rc_5shot
+    type: downstream
+
+  - label: arc_challenge_test_mc_5shot
+    type: downstream
+
+  - label: arc_easy_val_rc_5shot
+    type: downstream
+
+  - label: arc_easy_val_mc_5shot
+    type: downstream
+
+  - label: arc_easy_test_rc_5shot
+    type: downstream
+
+  - label: arc_easy_test_mc_5shot
+    type: downstream
+
+  - label: boolq_val_rc_5shot
+    type: downstream
+
+  - label: boolq_val_mc_5shot
+    type: downstream
+
+  - label: csqa_val_rc_5shot
+    type: downstream
+
+  - label: csqa_val_mc_5shot
+    type: downstream
+
+  - label: hellaswag_val_rc_5shot
+    type: downstream
+
+  - label: hellaswag_val_mc_5shot
+    type: downstream
+
+  - label: openbookqa_val_rc_5shot
+    type: downstream
+
+  - label: openbookqa_val_mc_5shot
+    type: downstream
+
+  - label: openbookqa_test_rc_5shot
+    type: downstream
+
+  - label: openbookqa_test_mc_5shot
+    type: downstream
+
+  - label: piqa_val_rc_5shot
+    type: downstream
+
+  - label: piqa_val_mc_5shot
+    type: downstream
+
+  - label: socialiqa_val_rc_5shot
+    type: downstream
+
+  - label: socialiqa_val_mc_5shot
+    type: downstream
+
+  - label: winogrande_val_rc_5shot
+    type: downstream
+
+  - label: winogrande_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_stem_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_stem_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_stem_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_stem_test_mc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_test_mc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_test_mc_5shot
+    type: downstream
+
+  - label: mmlu_other_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_other_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_other_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_other_test_mc_5shot
+    type: downstream
+
 data:
   pad_direction: right
   # generate_doc_lengths: true

configs/peteish7-weka.yaml

@@ -230,6 +230,126 @@ evaluators:
   - label: arc_easy_ppl
     type: downstream
 
+  - label: arc_challenge_val_rc_5shot
+    type: downstream
+
+  - label: arc_challenge_val_mc_5shot
+    type: downstream
+
+  - label: arc_challenge_test_rc_5shot
+    type: downstream
+
+  - label: arc_challenge_test_mc_5shot
+    type: downstream
+
+  - label: arc_easy_val_rc_5shot
+    type: downstream
+
+  - label: arc_easy_val_mc_5shot
+    type: downstream
+
+  - label: arc_easy_test_rc_5shot
+    type: downstream
+
+  - label: arc_easy_test_mc_5shot
+    type: downstream
+
+  - label: boolq_val_rc_5shot
+    type: downstream
+
+  - label: boolq_val_mc_5shot
+    type: downstream
+
+  - label: csqa_val_rc_5shot
+    type: downstream
+
+  - label: csqa_val_mc_5shot
+    type: downstream
+
+  - label: hellaswag_val_rc_5shot
+    type: downstream
+
+  - label: hellaswag_val_mc_5shot
+    type: downstream
+
+  - label: openbookqa_val_rc_5shot
+    type: downstream
+
+  - label: openbookqa_val_mc_5shot
+    type: downstream
+
+  - label: openbookqa_test_rc_5shot
+    type: downstream
+
+  - label: openbookqa_test_mc_5shot
+    type: downstream
+
+  - label: piqa_val_rc_5shot
+    type: downstream
+
+  - label: piqa_val_mc_5shot
+    type: downstream
+
+  - label: socialiqa_val_rc_5shot
+    type: downstream
+
+  - label: socialiqa_val_mc_5shot
+    type: downstream
+
+  - label: winogrande_val_rc_5shot
+    type: downstream
+
+  - label: winogrande_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_stem_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_stem_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_stem_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_stem_test_mc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_test_mc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_test_mc_5shot
+    type: downstream
+
+  - label: mmlu_other_val_rc_5shot
+    type: downstream
+
+  - label: mmlu_other_val_mc_5shot
+    type: downstream
+
+  - label: mmlu_other_test_rc_5shot
+    type: downstream
+
+  - label: mmlu_other_test_mc_5shot
+    type: downstream
+
 data:
   pad_direction: right
   # generate_doc_lengths: true

scripts/beaker/peteish/peteish7-eval.sh

@@ -55,6 +55,6 @@ torchrun \
       --save_overwrite \
       --device_eval_batch_size=16 \
       --wandb.group="peteish7" \
-      --load_path="/weka/oe-training-default/ai2-llm/checkpoints/OLMo-medium/peteish7"
+      --load_path="/weka/oe-training-default/ai2-llm/checkpoints/OLMo-medium/peteish7-anneal-from-928646-50B-no-warmup/step11931"
 
      # '--load_path=${path.last_checkpoint:${save_folder}}' \