use llm-foundry for ICL metrics (#287)

* use llm-foundry for ICL metrics * replaced InContextLearningQAAccuracy with InContextLearningGenerationExactMatchAccuracy, and removed the commented line for code evals * resolve lint check * Update requirements_test.txt * Update requirements_test.txt --------- Co-authored-by: Achal Dave <[email protected]> Co-authored-by: jmercat <[email protected]> Co-authored-by: sedrick-keh-tri <[email protected]>
mlfoundations · Jul 23, 2024 · 9bb92ef · 9bb92ef
1 parent c0f1319
commit 9bb92ef
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 7 deletions.
diff --git a/open_lm/utils/llm_foundry_wrapper.py b/open_lm/utils/llm_foundry_wrapper.py
@@ -4,14 +4,14 @@
 """Implements a Hugging Causal LM wrapped inside a :class:`.ComposerModel`."""
 
 from typing import Mapping, Union
-
-from composer.metrics.nlp import (
+from llmfoundry.eval.metrics.nlp import (
     InContextLearningLMAccuracy,
     InContextLearningLMExpectedCalibrationError,
     InContextLearningMCExpectedCalibrationError,
     InContextLearningMultipleChoiceAccuracy,
-    InContextLearningQAAccuracy,
-    InContextLearningCodeEvalAccuracy,
+    InContextLearningGenerationExactMatchAccuracy,
+)
+from composer.metrics.nlp import (
     LanguageCrossEntropy,
     LanguagePerplexity,
 )
@@ -33,10 +33,9 @@
     LanguagePerplexity(),
     InContextLearningLMAccuracy(),
     InContextLearningMultipleChoiceAccuracy(),
-    InContextLearningQAAccuracy(),
+    InContextLearningGenerationExactMatchAccuracy(),
     InContextLearningLMExpectedCalibrationError(),
     InContextLearningMCExpectedCalibrationError(),
-    InContextLearningCodeEvalAccuracy(),
 ]
 
 

diff --git a/requirements_test.txt b/requirements_test.txt
@@ -3,4 +3,4 @@ pytest-cov==3.0.0
 pytest-xdist==2.5.0
 pytest==7.0.1
 tensorboard==2.14.1
-llm-foundry==0.7.0
+llm-foundry==0.9.0