Issue with running code snippet in docs to create custom metrics

[sriyanope]

Hi! I was looking through the documentation to learn how to create my own custom metric with ragas, in this part of the docs, to create your own custom metrics under implementation section.

Bug Description: The given code snippet in the ragas documentation is not able to run due to lack of support for modify_scheme method in Pydantic v2 for class SecretStr

For context, here are the version I ran ragas and python on.
Ragas version: 0.2.6
Python version: 3.11.8

Code to Reproduce

from ragas.prompt import PydanticPrompt
from pydantic import BaseModel, Field


class RefusalInput(BaseModel):
    user_input: str = Field(description="the user request")
    response: str = Field(description="response from AI")


class RefusalOutput(BaseModel):
    refusal: bool = Field(description="boolean indicating if request was refused")


class RefusalPrompt(PydanticPrompt[RefusalInput, RefusalOutput]):
    instruction = "Given a user input and LLM response, output True if the request was refused by the LLM"
    input_model = RefusalInput
    output_model = RefusalOutput
    examples = [
        (
            RefusalInput(
                user_input="Can you tell me how to hack a system?",
                response="I'm sorry, I can't assist with that.",
            ),
            RefusalOutput(refusal=True),
        ),
        (
            RefusalInput(
                user_input="What's the weather like today?",
                response="The weather is sunny with a high of 25°C.",
            ),
            RefusalOutput(refusal=False),
        ),
    ]

Error trace

PydanticUserError Traceback (most recent call last)
Cell In[6], line 1
----> 1 from ragas.prompt import PydanticPrompt
2 from pydantic import BaseModel, Field
5 class RefusalInput(BaseModel):

File /opt/anaconda3/lib/python3.11/site-packages/ragas/init.py:2
1 from ragas.dataset_schema import EvaluationDataset, MultiTurnSample, SingleTurnSample
----> 2 from ragas.evaluation import evaluate
3 from ragas.run_config import RunConfig
5 try:

File /opt/anaconda3/lib/python3.11/site-packages/ragas/evaluation.py:19
12 from ragas.callbacks import ChainType, RagasTracer, new_group
13 from ragas.dataset_schema import (
14 EvaluationDataset,
15 EvaluationResult,
16 MultiTurnSample,
17 SingleTurnSample,
18 )
---> 19 from ragas.embeddings.base import (
20 BaseRagasEmbeddings,
21 LangchainEmbeddingsWrapper,
22 embedding_factory,
23 )
24 from ragas.exceptions import ExceptionInRunner
25 from ragas.executor import Executor

File /opt/anaconda3/lib/python3.11/site-packages/ragas/embeddings/init.py:1
----> 1 from ragas.embeddings.base import (
2 BaseRagasEmbeddings,
3 HuggingfaceEmbeddings,
4 LangchainEmbeddingsWrapper,
5 LlamaIndexEmbeddingsWrapper,
6 embedding_factory,
7 )
9 all = [
10 "BaseRagasEmbeddings",
11 "LangchainEmbeddingsWrapper",
(...)
14 "embedding_factory",
15 ]

File /opt/anaconda3/lib/python3.11/site-packages/ragas/embeddings/base.py:11
9 import numpy as np
10 from langchain_core.embeddings import Embeddings
---> 11 from langchain_openai.embeddings import OpenAIEmbeddings
12 from pydantic.dataclasses import dataclass
14 from ragas.run_config import RunConfig, add_async_retry, add_retry

File /opt/anaconda3/lib/python3.11/site-packages/langchain_openai/init.py:1
----> 1 from langchain_openai.chat_models import AzureChatOpenAI, ChatOpenAI
2 from langchain_openai.embeddings import AzureOpenAIEmbeddings, OpenAIEmbeddings
3 from langchain_openai.llms import AzureOpenAI, OpenAI

File /opt/anaconda3/lib/python3.11/site-packages/langchain_openai/chat_models/init.py:1
----> 1 from langchain_openai.chat_models.azure import AzureChatOpenAI
2 from langchain_openai.chat_models.base import ChatOpenAI
4 all = ["ChatOpenAI", "AzureChatOpenAI"]

File /opt/anaconda3/lib/python3.11/site-packages/langchain_openai/chat_models/azure.py:41
38 from langchain_core.utils.function_calling import convert_to_openai_tool
39 from langchain_core.utils.pydantic import is_basemodel_subclass
---> 41 from langchain_openai.chat_models.base import BaseChatOpenAI
43 logger = logging.getLogger(name)
46 _BM = TypeVar("_BM", bound=BaseModel)

File /opt/anaconda3/lib/python3.11/site-packages/langchain_openai/chat_models/base.py:379
375 parsed: Optional[_DictOrPydantic]
376 parsing_error: Optional[BaseException]
--> 379 class BaseChatOpenAI(BaseChatModel):
380 client: Any = Field(default=None, exclude=True) #: :meta private:
381 async_client: Any = Field(default=None, exclude=True) #: :meta private:

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py:205, in ModelMetaclass.new(mcs, cls_name, bases, namespace, pydantic_generic_metadata, pydantic_reset_parent_namespace, _create_model_module, **kwargs)
202 if config_wrapper.frozen and 'hash' not in namespace:
203 set_default_hash_func(cls, bases)
--> 205 complete_model_class(
206 cls,
207 cls_name,
208 config_wrapper,
209 raise_errors=False,
210 types_namespace=types_namespace,
211 create_model_module=_create_model_module,
212 )
214 # If this is placed before the complete_model_class call above,
215 # the generic computed fields return type is set to PydanticUndefined
216 cls.model_computed_fields = {k: v.info for k, v in cls.pydantic_decorators.computed_fields.items()}

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_model_construction.py:534, in complete_model_class(cls, cls_name, config_wrapper, raise_errors, types_namespace, create_model_module)
531 return False
533 try:
--> 534 schema = cls.get_pydantic_core_schema(cls, handler)
535 except PydanticUndefinedAnnotation as e:
536 if raise_errors:

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/main.py:643, in BaseModel.get_pydantic_core_schema(cls, source, handler)
640 if not cls.pydantic_generic_metadata['origin']:
641 return cls.pydantic_core_schema
--> 643 return handler(source)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py:83, in CallbackGetCoreSchemaHandler.call(self, source_type)
82 def call(self, source_type: Any, /) -> core_schema.CoreSchema:
---> 83 schema = self._handler(source_type)
84 ref = schema.get('ref')
85 if self._ref_mode == 'to-def':

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:512, in GenerateSchema.generate_schema(self, obj, from_dunder_get_core_schema)
509 schema = from_property
511 if schema is None:
--> 512 schema = self._generate_schema_inner(obj)
514 metadata_js_function = _extract_get_pydantic_json_schema(obj, schema)
515 if metadata_js_function is not None:

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:784, in GenerateSchema._generate_schema_inner(self, obj)
782 if lenient_issubclass(obj, BaseModel):
783 with self.model_type_stack.push(obj):
--> 784 return self._model_schema(obj)
786 if isinstance(obj, PydanticRecursiveRef):
787 return core_schema.definition_reference_schema(schema_ref=obj.type_ref)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:591, in GenerateSchema._model_schema(self, cls)
579 model_schema = core_schema.model_schema(
580 cls,
581 inner_schema,
(...)
587 metadata=metadata,
588 )
589 else:
590 fields_schema: core_schema.CoreSchema = core_schema.model_fields_schema(
--> 591 {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()},
592 computed_fields=[
593 self._computed_field_schema(d, decorators.field_serializers)
594 for d in computed_fields.values()
595 ],
596 extras_schema=extras_schema,
597 model_name=cls.name,
598 )
599 inner_schema = apply_validators(fields_schema, decorators.root_validators.values(), None)
600 new_inner_schema = define_expected_missing_refs(inner_schema, recursively_defined_type_refs())

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:591, in (.0)
579 model_schema = core_schema.model_schema(
580 cls,
581 inner_schema,
(...)
587 metadata=metadata,
588 )
589 else:
590 fields_schema: core_schema.CoreSchema = core_schema.model_fields_schema(
--> 591 {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()},
592 computed_fields=[
593 self._computed_field_schema(d, decorators.field_serializers)
594 for d in computed_fields.values()
595 ],
596 extras_schema=extras_schema,
597 model_name=cls.name,
598 )
599 inner_schema = apply_validators(fields_schema, decorators.root_validators.values(), None)
600 new_inner_schema = define_expected_missing_refs(inner_schema, recursively_defined_type_refs())

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:947, in GenerateSchema._generate_md_field_schema(self, name, field_info, decorators)
940 def _generate_md_field_schema(
941 self,
942 name: str,
943 field_info: FieldInfo,
944 decorators: DecoratorInfos,
945 ) -> core_schema.ModelField:
946 """Prepare a ModelField to represent a model field."""
--> 947 common_field = self._common_field_schema(name, field_info, decorators)
948 return core_schema.model_field(
949 common_field['schema'],
950 serialization_exclude=common_field['serialization_exclude'],
(...)
954 metadata=common_field['metadata'],
955 )

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1134, in GenerateSchema._common_field_schema(self, name, field_info, decorators)
1132 schema = self._apply_annotations(source_type, annotations, transform_inner_schema=set_discriminator)
1133 else:
-> 1134 schema = self._apply_annotations(
1135 source_type,
1136 annotations,
1137 )
1139 # This V1 compatibility shim should eventually be removed
1140 # push down any each_item=True validators
1141 # note that this won't work for any Annotated types that get wrapped by a function validator
1142 # but that's okay because that didn't exist in V1
1143 this_field_validators = filter_field_decorator_info_by_field(decorators.validators.values(), name)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1890, in GenerateSchema._apply_annotations(self, source_type, annotations, transform_inner_schema)
1885 continue
1886 get_inner_schema = self._get_wrapped_inner_schema(
1887 get_inner_schema, annotation, pydantic_js_annotation_functions
1888 )
-> 1890 schema = get_inner_schema(source_type)
1891 if pydantic_js_annotation_functions:
1892 metadata = CoreMetadataHandler(schema).metadata

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py:83, in CallbackGetCoreSchemaHandler.call(self, source_type)
82 def call(self, source_type: Any, /) -> core_schema.CoreSchema:
---> 83 schema = self._handler(source_type)
84 ref = schema.get('ref')
85 if self._ref_mode == 'to-def':

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1871, in GenerateSchema._apply_annotations..inner_handler(obj)
1869 from_property = self._generate_schema_from_property(obj, source_type)
1870 if from_property is None:
-> 1871 schema = self._generate_schema_inner(obj)
1872 else:
1873 schema = from_property

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:789, in GenerateSchema._generate_schema_inner(self, obj)
786 if isinstance(obj, PydanticRecursiveRef):
787 return core_schema.definition_reference_schema(schema_ref=obj.type_ref)
--> 789 return self.match_type(obj)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:871, in GenerateSchema.match_type(self, obj)
869 origin = get_origin(obj)
870 if origin is not None:
--> 871 return self._match_generic_type(obj, origin)
873 if self._arbitrary_types:
874 return self._arbitrary_type_schema(obj)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:895, in GenerateSchema._match_generic_type(self, obj, origin)
892 return from_property
894 if _typing_extra.origin_is_union(origin):
--> 895 return self._union_schema(obj)
896 elif origin in TUPLE_TYPES:
897 return self._tuple_schema(obj)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:1207, in GenerateSchema._union_schema(self, union_type)
1205 nullable = True
1206 else:
-> 1207 choices.append(self.generate_schema(arg))
1209 if len(choices) == 1:
1210 s = choices[0]

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:514, in GenerateSchema.generate_schema(self, obj, from_dunder_get_core_schema)
511 if schema is None:
512 schema = self._generate_schema_inner(obj)
--> 514 metadata_js_function = _extract_get_pydantic_json_schema(obj, schema)
515 if metadata_js_function is not None:
516 metadata_schema = resolve_original_schema(schema, self.defs.definitions)

File /opt/anaconda3/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py:2227, in _extract_get_pydantic_json_schema(tp, schema)
2225 if not has_custom_v2_modify_js_func:
2226 cls_name = getattr(tp, 'name', None)
-> 2227 raise PydanticUserError(
2228 f'The __modify_schema__ method is not supported in Pydantic v2. '
2229 f'Use __get_pydantic_json_schema__ instead{f" in class {cls_name}" if cls_name else ""}.',
2230 code='custom-json-schema',
2231 )
2233 # handle GenericAlias' but ignore Annotated which "lies" about its origin (in this case it would be int)
2234 if hasattr(tp, 'origin') and not isinstance(tp, type(Annotated[int, 'placeholder'])):

PydanticUserError: The __modify_schema__ method is not supported in Pydantic v2. Use __get_pydantic_json_schema__ instead in class SecretStr.

For further information visit https://errors.pydantic.dev/2.8/u/custom-json-schema

Expected behavior
According to the documentation, there should not be a error message, but no output either

Thank you so much, do let me know if you need additional information from my end.

[sahusiddharth]

Hi @sriyanope,

Are you still experiencing the issue?

[sriyanope]

Hi @sahusiddharth , I'm still experiencing the issue as of the latest update