diff --git a/libs/langchain/langchain/evaluation/comparison/eval_chain.py b/libs/langchain/langchain/evaluation/comparison/eval_chain.py
index b0fd1e27a2737..ada4bffe21a77 100644
--- a/libs/langchain/langchain/evaluation/comparison/eval_chain.py
+++ b/libs/langchain/langchain/evaluation/comparison/eval_chain.py
@@ -3,10 +3,8 @@
 
 import logging
 import re
-from typing import Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-from langchain_community.chat_models.azure_openai import AzureChatOpenAI
-from langchain_community.chat_models.openai import ChatOpenAI
 from langchain_core.callbacks.manager import Callbacks
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.output_parsers import BaseOutputParser
@@ -27,6 +25,10 @@
 from langchain.evaluation.schema import LLMEvalChain, PairwiseStringEvaluator
 from langchain.schema import RUN_KEY
 
+if TYPE_CHECKING:
+    from langchain_community.chat_models.azure_openai import AzureChatOpenAI
+    from langchain_community.chat_models.openai import ChatOpenAI
+
 logger = logging.getLogger(__name__)
 
 _FIND_DOUBLE_BRACKETS = re.compile(r"\[\[(.*?)\]\]")
diff --git a/libs/langchain/langchain/evaluation/criteria/eval_chain.py b/libs/langchain/langchain/evaluation/criteria/eval_chain.py
index 4a18aa081b575..9df3853195850 100644
--- a/libs/langchain/langchain/evaluation/criteria/eval_chain.py
+++ b/libs/langchain/langchain/evaluation/criteria/eval_chain.py
@@ -193,7 +193,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
 
     Examples
     --------
-    >>> from langchain_community.chat_models import ChatAnthropic
+    >>> from langchain_anthropic import ChatAnthropic
     >>> from langchain.evaluation.criteria import CriteriaEvalChain
     >>> llm = ChatAnthropic(temperature=0)
     >>> criteria = {"my-custom-criterion": "Is the submission the most amazing ever?"}
@@ -205,7 +205,7 @@ class CriteriaEvalChain(StringEvaluator, LLMEvalChain, LLMChain):
         'score': 0,
     }
 
-    >>> from langchain_community.chat_models import ChatOpenAI
+    >>> from langchain_openai import ChatOpenAI
     >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
     >>> llm = ChatOpenAI(model="gpt-4", temperature=0)
     >>> criteria = "correctness"
@@ -344,7 +344,7 @@ def from_llm(
 
         Examples
         --------
-        >>> from langchain_community.llms import OpenAI
+        >>> from langchain_openai import OpenAI
         >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
         >>> llm = OpenAI()
         >>> criteria = {
@@ -432,7 +432,7 @@ def _evaluate_strings(
 
         Examples
         --------
-        >>> from langchain_community.llms import OpenAI
+        >>> from langchain_openai import OpenAI
         >>> from langchain.evaluation.criteria import CriteriaEvalChain
         >>> llm = OpenAI()
         >>> criteria = "conciseness"
@@ -487,7 +487,7 @@ async def _aevaluate_strings(
 
         Examples
         --------
-        >>> from langchain_community.llms import OpenAI
+        >>> from langchain_openai import OpenAI
         >>> from langchain.evaluation.criteria import CriteriaEvalChain
         >>> llm = OpenAI()
         >>> criteria = "conciseness"
@@ -568,7 +568,7 @@ def from_llm(
 
         Examples
         --------
-        >>> from langchain_community.llms import OpenAI
+        >>> from langchain_openai import OpenAI
         >>> from langchain.evaluation.criteria import LabeledCriteriaEvalChain
         >>> llm = OpenAI()
         >>> criteria = {
diff --git a/libs/langchain/langchain/evaluation/loading.py b/libs/langchain/langchain/evaluation/loading.py
index fbb1732585493..be762145890db 100644
--- a/libs/langchain/langchain/evaluation/loading.py
+++ b/libs/langchain/langchain/evaluation/loading.py
@@ -1,7 +1,6 @@
 """Loading datasets and evaluators."""
 from typing import Any, Dict, List, Optional, Sequence, Type, Union
 
-from langchain_community.chat_models.openai import ChatOpenAI
 from langchain_core.language_models import BaseLanguageModel
 
 from langchain.chains.base import Chain
@@ -131,6 +130,20 @@ def load_evaluator(
     evaluator_cls = _EVALUATOR_MAP[evaluator]
     if issubclass(evaluator_cls, LLMEvalChain):
         try:
+            try:
+                from langchain_openai import ChatOpenAI
+            except ImportError:
+                try:
+                    from langchain_community.chat_models.openai import ChatOpenAI
+                except ImportError:
+                    raise ImportError(
+                        "Could not import langchain_openai or fallback onto "
+                        "langchain_community. Please install langchain_openai "
+                        "or specify a language model explicitly."
+                        "It's recommended to install langchain_openai AND "
+                        "specify a language model explicitly."
+                    )
+
             llm = llm or ChatOpenAI(  # type: ignore[call-arg]
                 model="gpt-4", model_kwargs={"seed": 42}, temperature=0
             )