From 2cf3e85639edfbdb586d304362c3f9eee1c879a2 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Mon, 22 Apr 2024 17:05:37 -0400 Subject: [PATCH 1/4] x --- .../langchain/evaluation/comparison/eval_chain.py | 8 +++++--- .../langchain/evaluation/criteria/eval_chain.py | 12 ++++++------ libs/langchain/langchain/evaluation/loading.py | 15 ++++++++++++++- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/libs/langchain/langchain/evaluation/comparison/eval_chain.py b/libs/langchain/langchain/evaluation/comparison/eval_chain.py index b0fd1e27a2737..ada4bffe21a77 100644 --- a/libs/langchain/langchain/evaluation/comparison/eval_chain.py +++ b/libs/langchain/langchain/evaluation/comparison/eval_chain.py @@ -3,10 +3,8 @@ import logging import re -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from langchain_community.chat_models.azure_openai import AzureChatOpenAI -from langchain_community.chat_models.openai import ChatOpenAI from langchain_core.callbacks.manager import Callbacks from langchain_core.language_models import BaseLanguageModel from langchain_core.output_parsers import BaseOutputParser @@ -27,6 +25,10 @@ from langchain.evaluation.schema import LLMEvalChain, PairwiseStringEvaluator from langchain.schema import RUN_KEY +if TYPE_CHECKING: + from langchain_community.chat_models.azure_openai import AzureChatOpenAI + from langchain_community.chat_models.openai import ChatOpenAI + logger = logging.getLogger(__name__) _FIND_DOUBLE_BRACKETS = re.compile(r"\[\[(.*?)\]\]") diff --git a/libs/langchain/langchain/evaluation/criteria/eval_chain.py b/libs/langchain/langchain/evaluation/criteria/eval_chain.py index 4a18aa081b575..9df3853195850 100644 --- a/libs/langchain/langchain/evaluation/criteria/eval_chain.py +++ b/libs/langchain/langchain/evaluation/criteria/eval_chain.py @@ -193,7 +193,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): Examples -------- - >>> from langchain_community.chat_models import ChatAnthropic + >>> from langchain_anthropic import ChatAnthropic >>> from langchain.evaluation.criteria import CriteriaEvalChain >>> llm = ChatAnthropic(temperature=0) >>> criteria = {"my-custom-criterion": "Is the submission the most amazing ever?"} @@ -205,7 +205,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain): 'score': 0, } - >>> from langchain_community.chat_models import ChatOpenAI + >>> from langchain_openai import ChatOpenAI >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain >>> llm = ChatOpenAI(model="gpt-4", temperature=0) >>> criteria = "correctness" @@ -344,7 +344,7 @@ def from_llm( Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain >>> llm = OpenAI() >>> criteria = { @@ -432,7 +432,7 @@ def _evaluate_strings( Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import CriteriaEvalChain >>> llm = OpenAI() >>> criteria = "conciseness" @@ -487,7 +487,7 @@ async def _aevaluate_strings( Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import CriteriaEvalChain >>> llm = OpenAI() >>> criteria = "conciseness" @@ -568,7 +568,7 @@ def from_llm( Examples -------- - >>> from langchain_community.llms import OpenAI + >>> from langchain_openai import OpenAI >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain >>> llm = OpenAI() >>> criteria = { diff --git a/libs/langchain/langchain/evaluation/loading.py b/libs/langchain/langchain/evaluation/loading.py index fbb1732585493..be762145890db 100644 --- a/libs/langchain/langchain/evaluation/loading.py +++ b/libs/langchain/langchain/evaluation/loading.py @@ -1,7 +1,6 @@ """Loading datasets and evaluators.""" from typing import Any, Dict, List, Optional, Sequence, Type, Union -from langchain_community.chat_models.openai import ChatOpenAI from langchain_core.language_models import BaseLanguageModel from langchain.chains.base import Chain @@ -131,6 +130,20 @@ def load_evaluator( evaluator_cls = _EVALUATOR_MAP[evaluator] if issubclass(evaluator_cls, LLMEvalChain): try: + try: + from langchain_openai import ChatOpenAI + except ImportError: + try: + from langchain_community.chat_models.openai import ChatOpenAI + except ImportError: + raise ImportError( + "Could not import langchain_openai or fallback onto " + "langchain_community. Please install langchain_openai " + "or specify a language model explicitly." + "It's recommended to install langchain_openai AND " + "specify a language model explicitly." + ) + llm = llm or ChatOpenAI( # type: ignore[call-arg] model="gpt-4", model_kwargs={"seed": 42}, temperature=0 ) From f391e942cbe82761bc4c811439000afeaebac832 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 23 Apr 2024 10:21:21 -0400 Subject: [PATCH 2/4] Update libs/langchain/langchain/evaluation/loading.py Co-authored-by: ccurme --- libs/langchain/langchain/evaluation/loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain/langchain/evaluation/loading.py b/libs/langchain/langchain/evaluation/loading.py index be762145890db..27a8f348e48f0 100644 --- a/libs/langchain/langchain/evaluation/loading.py +++ b/libs/langchain/langchain/evaluation/loading.py @@ -139,7 +139,7 @@ def load_evaluator( raise ImportError( "Could not import langchain_openai or fallback onto " "langchain_community. Please install langchain_openai " - "or specify a language model explicitly." + "or specify a language model explicitly. " "It's recommended to install langchain_openai AND " "specify a language model explicitly." ) From c85b4b841d9e3556072f77574330898439850afa Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 23 Apr 2024 10:54:29 -0400 Subject: [PATCH 3/4] x --- .../langchain/evaluation/comparison/eval_chain.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/libs/langchain/langchain/evaluation/comparison/eval_chain.py b/libs/langchain/langchain/evaluation/comparison/eval_chain.py index ada4bffe21a77..a9bc671f4e867 100644 --- a/libs/langchain/langchain/evaluation/comparison/eval_chain.py +++ b/libs/langchain/langchain/evaluation/comparison/eval_chain.py @@ -25,10 +25,6 @@ from langchain.evaluation.schema import LLMEvalChain, PairwiseStringEvaluator from langchain.schema import RUN_KEY -if TYPE_CHECKING: - from langchain_community.chat_models.azure_openai import AzureChatOpenAI - from langchain_community.chat_models.openai import ChatOpenAI - logger = logging.getLogger(__name__) _FIND_DOUBLE_BRACKETS = re.compile(r"\[\[(.*?)\]\]") @@ -256,10 +252,8 @@ def from_llm( ValueError: If the input variables are not as expected. """ - if not ( - isinstance(llm, (ChatOpenAI, AzureChatOpenAI)) - and llm.model_name.startswith("gpt-4") - ): + # Check if the model is GPT-4 if not raise a warning + if not hasattr(llm, "model_name") or not llm.model_name.startswith("gpt-4"): logger.warning( "This chain was only tested with GPT-4. \ Performance may be significantly worse with other models." From 0731d0083b5de32e3efd620a28e893c6f93da07c Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 23 Apr 2024 10:57:11 -0400 Subject: [PATCH 4/4] x --- libs/langchain/langchain/evaluation/comparison/eval_chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain/langchain/evaluation/comparison/eval_chain.py b/libs/langchain/langchain/evaluation/comparison/eval_chain.py index a9bc671f4e867..2bcf5632612b7 100644 --- a/libs/langchain/langchain/evaluation/comparison/eval_chain.py +++ b/libs/langchain/langchain/evaluation/comparison/eval_chain.py @@ -3,7 +3,7 @@ import logging import re -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from langchain_core.callbacks.manager import Callbacks from langchain_core.language_models import BaseLanguageModel