diff --git a/pyproject.toml b/pyproject.toml index 111927e85..76e57481a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ dependencies = [ "appdirs", "pydantic>=2", "openai>1", - "pysbd>=0.3.4", "diskcache>=5.6.3", ] dynamic = ["version", "readme"] diff --git a/src/ragas/metrics/_answer_correctness.py b/src/ragas/metrics/_answer_correctness.py index a5c10e206..8242e9ffe 100644 --- a/src/ragas/metrics/_answer_correctness.py +++ b/src/ragas/metrics/_answer_correctness.py @@ -10,9 +10,9 @@ from ragas.dataset_schema import SingleTurnSample from ragas.metrics._answer_similarity import AnswerSimilarity from ragas.metrics._faithfulness import ( - FaithfulnessStatements, - HasSegmentMethod, - LongFormAnswerPrompt, + StatementGeneratorInput, + StatementGeneratorOutput, + StatementGeneratorPrompt, ) from ragas.metrics.base import ( MetricOutputType, @@ -20,7 +20,6 @@ MetricWithEmbeddings, MetricWithLLM, SingleTurnMetric, - get_segmenter, ) from ragas.metrics.utils import fbeta_score from ragas.prompt import PydanticPrompt @@ -29,9 +28,6 @@ if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks - from ragas.metrics._faithfulness import SentencesSimplified - - logger = logging.getLogger(__name__) @@ -166,13 +162,12 @@ class AnswerCorrectness(MetricWithLLM, MetricWithEmbeddings, SingleTurnMetric): ) output_type = MetricOutputType.CONTINUOUS correctness_prompt: PydanticPrompt = field(default_factory=CorrectnessClassifier) - long_form_answer_prompt: PydanticPrompt = field( - default_factory=LongFormAnswerPrompt + statement_generator_prompt: PydanticPrompt = field( + default_factory=StatementGeneratorPrompt ) weights: list[float] = field(default_factory=lambda: [0.75, 0.25]) beta: float = 1.0 answer_similarity: t.Optional[AnswerSimilarity] = None - sentence_segmenter: t.Optional[HasSegmentMethod] = None max_retries: int = 1 def __post_init__(self): @@ -185,10 +180,6 @@ def __post_init__(self): if not all([w >= 0 for w in self.weights]): raise ValueError("Weights must be non-negative") - if self.sentence_segmenter is None: - language = self.long_form_answer_prompt.language - self.sentence_segmenter = get_segmenter(language=language, clean=False) - if type(self.beta) is not float: raise ValueError( "Beta must be a float. A beta > 1 gives more weight to recall, while beta < 1 favors precision." @@ -210,25 +201,17 @@ def _compute_statement_presence( async def _create_simplified_statements( self, question: str, text: str, callbacks: Callbacks - ) -> SentencesSimplified: - assert self.sentence_segmenter is not None, "sentence_segmenter is not set" + ) -> StatementGeneratorOutput: assert self.llm is not None, "llm is not set" - sentences = self.sentence_segmenter.segment(text) - sentences_with_index = { - i: sentence - for i, sentence in enumerate(sentences) - if sentence.strip().endswith(".") - } - - statements_simplified = await self.long_form_answer_prompt.generate( + prompt_input = StatementGeneratorInput(question=question, answer=text) + statements = await self.statement_generator_prompt.generate( llm=self.llm, - data=FaithfulnessStatements( - question=question, answer=text, sentences=sentences_with_index - ), + data=prompt_input, callbacks=callbacks, ) - return statements_simplified + + return statements async def _single_turn_ascore( self, sample: SingleTurnSample, callbacks: Callbacks @@ -244,13 +227,11 @@ async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float: question = row["user_input"] statements: t.Dict[str, t.List[str]] = {} for item in ["response", "reference"]: - simplified_statements = await self._create_simplified_statements( + statements_x = await self._create_simplified_statements( question, row[item], callbacks ) - _statements_unwrapped = [] - for component in simplified_statements.sentences: - _statements_unwrapped.extend(component.simpler_statements) - statements[item] = _statements_unwrapped + statements_x = statements_x.statements + statements[item] = statements_x if not all([val == [] for val in statements.values()]): ground_truth = [statement for statement in statements["reference"]] diff --git a/src/ragas/metrics/_bleu_score.py b/src/ragas/metrics/_bleu_score.py index c3ed2a48a..71505a4bf 100644 --- a/src/ragas/metrics/_bleu_score.py +++ b/src/ragas/metrics/_bleu_score.py @@ -4,8 +4,7 @@ from langchain_core.callbacks import Callbacks from ragas.dataset_schema import SingleTurnSample -from ragas.metrics._faithfulness import HasSegmentMethod -from ragas.metrics.base import MetricType, SingleTurnMetric, get_segmenter +from ragas.metrics.base import MetricType, SingleTurnMetric from ragas.run_config import RunConfig @@ -15,7 +14,6 @@ class BleuScore(SingleTurnMetric): _required_columns: t.Dict[MetricType, t.Set[str]] = field( default_factory=lambda: {MetricType.SINGLE_TURN: {"reference", "response"}} ) - sentence_segmenter: t.Optional[HasSegmentMethod] = None language: str = "english" def __post_init__(self): @@ -25,8 +23,6 @@ def __post_init__(self): raise ImportError( "sacrebleu is required for bleu score. Please install it using `pip install sacrebleu`" ) - if not self.sentence_segmenter: - self.sentence_segmenter = get_segmenter(language=self.language, clean=False) self.corpus_bleu = corpus_bleu def init(self, run_config: RunConfig): @@ -35,12 +31,13 @@ def init(self, run_config: RunConfig): async def _single_turn_ascore( self, sample: SingleTurnSample, callbacks: Callbacks ) -> float: - assert ( - self.sentence_segmenter is not None - ), "Sentence segmenter is not initialized" - reference_sentences = self.sentence_segmenter.segment(sample.reference) - response_sentences = self.sentence_segmenter.segment(sample.response) + reference, response = sample.reference, sample.response + assert isinstance(reference, str), "BleuScore expects a valid reference string" + assert isinstance(response, str), "BleuScore expects a valid response string" + + reference_sentences = reference.split(". ") + response_sentences = response.split(". ") reference = [[reference] for reference in reference_sentences] response = response_sentences diff --git a/src/ragas/metrics/_factual_correctness.py b/src/ragas/metrics/_factual_correctness.py index f5b8b70e9..81507d799 100644 --- a/src/ragas/metrics/_factual_correctness.py +++ b/src/ragas/metrics/_factual_correctness.py @@ -9,17 +9,12 @@ from numpy.typing import NDArray from pydantic import BaseModel, Field -from ragas.metrics._faithfulness import ( - HasSegmentMethod, - NLIStatementInput, - NLIStatementPrompt, -) +from ragas.metrics._faithfulness import NLIStatementInput, NLIStatementPrompt from ragas.metrics.base import ( MetricOutputType, MetricType, MetricWithLLM, SingleTurnMetric, - get_segmenter, ) from ragas.metrics.utils import fbeta_score from ragas.prompt import PydanticPrompt @@ -35,11 +30,10 @@ class ClaimDecompositionInput(BaseModel): response: str = Field(..., title="Response") - sentences: t.List[str] = Field(..., title="Sentences from response") class ClaimDecompositionOutput(BaseModel): - decomposed_claims: t.List[t.List[str]] = Field(..., title="Decomposed Claims") + claims: t.List[str] = Field(..., title="Decomposed Claims") # Define an enum for decomposition types @@ -52,21 +46,16 @@ class DecompositionType(Enum): # Example input data example1_input = ClaimDecompositionInput( - response="Charles Babbage was a French mathematician, philosopher, and food critic.", - sentences=[ - "Charles Babbage was a French mathematician, philosopher, and food critic." - ], + response="Charles Babbage was a French mathematician, philosopher, and food critic." ) -# Define the examples using the new structure +# Define the examples using the Pydantic structure claim_decomposition_examples = { DecompositionType.LOW_ATOMICITY_LOW_COVERAGE: [ ( example1_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Charles Babbage was a mathematician and philosopher."] - ] + claims=["Charles Babbage was a mathematician and philosopher."] ), ) ], @@ -74,10 +63,8 @@ class DecompositionType(Enum): ( example1_input, ClaimDecompositionOutput( - decomposed_claims=[ - [ - "Charles Babbage was a French mathematician, philosopher, and food critic." - ] + claims=[ + "Charles Babbage was a French mathematician, philosopher, and food critic." ] ), ) @@ -86,9 +73,9 @@ class DecompositionType(Enum): ( example1_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Charles Babbage was a mathematician."], - ["Charles Babbage was a philosopher."], + claims=[ + "Charles Babbage was a mathematician.", + "Charles Babbage was a philosopher.", ] ), ) @@ -97,11 +84,11 @@ class DecompositionType(Enum): ( example1_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Charles Babbage was a mathematician."], - ["Charles Babbage was a philosopher."], - ["Charles Babbage was a food critic."], - ["Charles Babbage was French."], + claims=[ + "Charles Babbage was a mathematician.", + "Charles Babbage was a philosopher.", + "Charles Babbage was a food critic.", + "Charles Babbage was French.", ] ), ) @@ -110,11 +97,7 @@ class DecompositionType(Enum): # Example input data with two sentences example2_input = ClaimDecompositionInput( - response="Albert Einstein was a German theoretical physicist. He developed the theory of relativity and also contributed to the development of quantum mechanics.", - sentences=[ - "Albert Einstein was a German theoretical physicist.", - "He developed the theory of relativity and also contributed to the development of quantum mechanics.", - ], + response="Albert Einstein was a German theoretical physicist. He developed the theory of relativity and also contributed to the development of quantum mechanics." ) # Adding examples to the dictionary with different decomposition types @@ -122,11 +105,9 @@ class DecompositionType(Enum): ( example2_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Albert Einstein was a German physicist."], - [ - "Albert Einstein developed relativity and contributed to quantum mechanics." - ], + claims=[ + "Albert Einstein was a German physicist.", + "Albert Einstein developed relativity and contributed to quantum mechanics.", ] ), ) @@ -136,11 +117,9 @@ class DecompositionType(Enum): ( example2_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Albert Einstein was a German theoretical physicist."], - [ - "Albert Einstein developed the theory of relativity and also contributed to the development of quantum mechanics." - ], + claims=[ + "Albert Einstein was a German theoretical physicist.", + "Albert Einstein developed the theory of relativity and also contributed to the development of quantum mechanics.", ] ), ) @@ -150,9 +129,9 @@ class DecompositionType(Enum): ( example2_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Albert Einstein was a German theoretical physicist."], - ["Albert Einstein developed the theory of relativity."], + claims=[ + "Albert Einstein was a German theoretical physicist.", + "Albert Einstein developed the theory of relativity.", ] ), ) @@ -162,12 +141,10 @@ class DecompositionType(Enum): ( example2_input, ClaimDecompositionOutput( - decomposed_claims=[ - ["Albert Einstein was a German theoretical physicist."], - [ - "Albert Einstein developed the theory of relativity.", - "Albert Einstein contributed to the development of quantum mechanics.", - ], + claims=[ + "Albert Einstein was a German theoretical physicist.", + "Albert Einstein developed the theory of relativity.", + "Albert Einstein contributed to the development of quantum mechanics.", ] ), ) @@ -218,7 +195,6 @@ class FactualCorrectness(MetricWithLLM, SingleTurnMetric): coverage: t.Literal["low", "high"] = "low" claim_decomposition_prompt: PydanticPrompt = ClaimDecompositionPrompt() nli_prompt: PydanticPrompt = NLIStatementPrompt() - sentence_segmenter: t.Optional[HasSegmentMethod] = None language: str = "english" def __post_init__(self): @@ -232,8 +208,6 @@ def __post_init__(self): logger.warning( f"No examples found for the atomicity and coverage level: {value}" ) - if not self.sentence_segmenter: - self.sentence_segmenter = get_segmenter(language=self.language, clean=False) if type(self.beta) is not float: raise ValueError( @@ -244,20 +218,12 @@ async def decompose_claims( self, response: str, callbacks: Callbacks ) -> t.List[str]: assert self.llm is not None, "LLM must be set" - assert ( - self.sentence_segmenter is not None - ), "Sentence segmenter is not initialized" - sentences = self.sentence_segmenter.segment(response) - assert isinstance(sentences, list), "Segmenter must return a list of sentences" - prompt_input = ClaimDecompositionInput(response=response, sentences=sentences) + prompt_input = ClaimDecompositionInput(response=response) result = await self.claim_decomposition_prompt.generate( data=prompt_input, llm=self.llm, callbacks=callbacks ) - claims_list = [ - claim for claim_list in result.decomposed_claims for claim in claim_list - ] - return claims_list + return result.claims async def verify_claims( self, premise: str, hypothesis_list: t.List[str], callbacks: Callbacks diff --git a/src/ragas/metrics/_faithfulness.py b/src/ragas/metrics/_faithfulness.py index f689bc1a8..4618dcc4f 100644 --- a/src/ragas/metrics/_faithfulness.py +++ b/src/ragas/metrics/_faithfulness.py @@ -13,77 +13,46 @@ MetricType, MetricWithLLM, SingleTurnMetric, - get_segmenter, ) from ragas.prompt import PydanticPrompt if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks - -class HasSegmentMethod(t.Protocol): - def segment(self, text) -> t.Any: ... - - logger = logging.getLogger(__name__) -class FaithfulnessStatements(BaseModel): +class StatementGeneratorInput(BaseModel): question: str = Field(description="The question to answer") answer: str = Field(description="The answer to the question") - sentences: t.Dict[int, str] = Field( - description="A mapping of sentence index to the sentence" - ) - - -class SentenceComponents(BaseModel): - sentence_index: int = Field(description="The index of the sentence") - simpler_statements: t.List[str] = Field( - description="A list of simpler statements that can be directly inferred from the context" - ) - -class SentencesSimplified(BaseModel): - sentences: t.List[SentenceComponents] = Field( - description="A list of sentences and their simpler versions" - ) - - -# examples -example_input_1 = FaithfulnessStatements( - question="Who was Albert Einstein and what is he best known for?", - answer="He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.", - sentences={ - 0: "He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time.", - 1: "He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.", - }, -) -example_output_1 = SentencesSimplified( - sentences=[ - SentenceComponents( - sentence_index=0, - simpler_statements=[ - "Albert Einstein was a German-born theoretical physicist.", - "Albert Einstein is recognized as one of the greatest and most influential physicists of all time.", - ], - ), - SentenceComponents( - sentence_index=1, - simpler_statements=[ - "Albert Einstein was best known for developing the theory of relativity.", - "Albert Einstein also made important contributions to the development of the theory of quantum mechanics.", - ], - ), - ] -) +class StatementGeneratorOutput(BaseModel): + statements: t.List[str] = Field(description="The generated statements") -class LongFormAnswerPrompt(PydanticPrompt[FaithfulnessStatements, SentencesSimplified]): +class StatementGeneratorPrompt( + PydanticPrompt[StatementGeneratorInput, StatementGeneratorOutput] +): instruction = "Given a question, an answer, and sentences from the answer analyze the complexity of each sentence given under 'sentences' and break down each sentence into one or more fully understandable statements while also ensuring no pronouns are used in each statement. Format the outputs in JSON." - input_model = FaithfulnessStatements - output_model = SentencesSimplified - examples = [(example_input_1, example_output_1)] + input_model = StatementGeneratorInput + output_model = StatementGeneratorOutput + examples = [ + ( + StatementGeneratorInput( + question="Who was Albert Einstein and what is he best known for?", + answer="He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.", + ), + StatementGeneratorOutput( + statements=[ + "Albert Einstein was a German-born theoretical physicist.", + "Albert Einstein is recognized as one of the greatest and most influential physicists of all time.", + "Albert Einstein was best known for developing the theory of relativity.", + "Albert Einstein also made important contributions to the development of the theory of quantum mechanics.", + ] + ), + ) + ] class StatementFaithfulnessAnswer(BaseModel): @@ -174,23 +143,19 @@ class Faithfulness(MetricWithLLM, SingleTurnMetric): } ) output_type: t.Optional[MetricOutputType] = MetricOutputType.CONTINUOUS - nli_statements_message: PydanticPrompt = field(default_factory=NLIStatementPrompt) - statement_prompt: PydanticPrompt = field(default_factory=LongFormAnswerPrompt) - sentence_segmenter: t.Optional[HasSegmentMethod] = None + nli_statements_prompt: PydanticPrompt = field(default_factory=NLIStatementPrompt) + statement_generator_prompt: PydanticPrompt = field( + default_factory=StatementGeneratorPrompt + ) max_retries: int = 1 - def __post_init__(self): - if self.sentence_segmenter is None: - language = self.nli_statements_message.language - self.sentence_segmenter = get_segmenter(language=language, clean=False) - async def _create_verdicts( self, row: t.Dict, statements: t.List[str], callbacks: Callbacks ) -> NLIStatementOutput: assert self.llm is not None, "llm must be set to compute score" contexts_str: str = "\n".join(row["retrieved_contexts"]) - verdicts = await self.nli_statements_message.generate( + verdicts = await self.nli_statements_prompt.generate( data=NLIStatementInput(context=contexts_str, statements=statements), llm=self.llm, callbacks=callbacks, @@ -200,22 +165,19 @@ async def _create_verdicts( async def _create_statements( self, row: t.Dict, callbacks: Callbacks - ) -> SentencesSimplified: + ) -> StatementGeneratorOutput: assert self.llm is not None, "llm is not set" - assert self.sentence_segmenter is not None, "sentence_segmenter is not set" text, question = row["response"], row["user_input"] - sentences = self.sentence_segmenter.segment(text) - sentences_with_index = {i: sentence for i, sentence in enumerate(sentences)} - statements_simplified = await self.statement_prompt.generate( + prompt_input = StatementGeneratorInput(question=question, answer=text) + statements = await self.statement_generator_prompt.generate( llm=self.llm, - data=FaithfulnessStatements( - question=question, answer=text, sentences=sentences_with_index - ), + data=prompt_input, callbacks=callbacks, ) - return statements_simplified + + return statements def _compute_score(self, answers: NLIStatementOutput): # check the verdicts and compute the score @@ -243,15 +205,11 @@ async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float: """ assert self.llm is not None, "LLM is not set" - statements_simplified = await self._create_statements(row, callbacks) - if statements_simplified is None: + statements = await self._create_statements(row, callbacks) + statements = statements.statements + if statements == []: return np.nan - # unwrap the statements - statements = [] - for component in statements_simplified.sentences: - statements.extend(component.simpler_statements) - verdicts = await self._create_verdicts(row, statements, callbacks) return self._compute_score(verdicts) @@ -298,15 +256,11 @@ async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float: """ assert self.llm is not None, "LLM is not set" - statements_simplified = await self._create_statements(row, callbacks) - if len(statements_simplified.sentences) == 0: + statements = await self._create_statements(row, callbacks) + statements = statements.statements + if statements == []: return np.nan - statements = [] - for components in statements_simplified.sentences: - statements.extend(components.simpler_statements) - assert isinstance(statements, t.List), "statements must be a list" - scores = [] pairs = self._create_pairs(row, statements) for input_pairs in self._create_batch(pairs): # to avoid OOM diff --git a/src/ragas/metrics/_noise_sensitivity.py b/src/ragas/metrics/_noise_sensitivity.py index 7856f0426..685577dd7 100644 --- a/src/ragas/metrics/_noise_sensitivity.py +++ b/src/ragas/metrics/_noise_sensitivity.py @@ -8,18 +8,16 @@ from ragas.dataset_schema import SingleTurnSample from ragas.metrics._faithfulness import ( - FaithfulnessStatements, - HasSegmentMethod, - LongFormAnswerPrompt, NLIStatementInput, NLIStatementPrompt, + StatementGeneratorInput, + StatementGeneratorPrompt, ) from ragas.metrics.base import ( MetricOutputType, MetricType, MetricWithLLM, SingleTurnMetric, - get_segmenter, ) from ragas.prompt import PydanticPrompt @@ -45,15 +43,14 @@ class NoiseSensitivity(MetricWithLLM, SingleTurnMetric): } ) output_type: t.Optional[MetricOutputType] = MetricOutputType.CONTINUOUS - nli_statements_message: PydanticPrompt = field(default_factory=NLIStatementPrompt) - statement_prompt: PydanticPrompt = field(default_factory=LongFormAnswerPrompt) - sentence_segmenter: t.Optional[HasSegmentMethod] = None + nli_statements_prompt: PydanticPrompt = field(default_factory=NLIStatementPrompt) + statement_generator_prompt: PydanticPrompt = field( + default_factory=StatementGeneratorPrompt + ) max_retries: int = 1 def __post_init__(self): - if self.sentence_segmenter is None: - language = self.nli_statements_message.language - self.sentence_segmenter = get_segmenter(language=language, clean=False) + if self.focus not in {"relevant", "irrelevant"}: raise ValueError( f"Invalid argument passed for 'focus': {self.focus}. Must be 'relevant' or 'irrelevant'." @@ -65,7 +62,7 @@ async def _evaluate_statement_faithfulness( ) -> t.List[int]: assert self.llm is not None, "LLM is not set" - verdicts = await self.nli_statements_message.generate( + verdicts = await self.nli_statements_prompt.generate( data=NLIStatementInput(context=context, statements=statements), llm=self.llm, callbacks=callbacks, @@ -80,24 +77,13 @@ async def _decompose_answer_into_statements( self, text: str, question: str, callbacks: Callbacks ) -> t.List[str]: assert self.llm is not None, "LLM is not set" - assert self.sentence_segmenter is not None, "sentence_segmenter is not set" - sentences = self.sentence_segmenter.segment(text) - sentences_with_index = {i: sentence for i, sentence in enumerate(sentences)} - - statements_simplified = await self.statement_prompt.generate( + statements = await self.statement_generator_prompt.generate( llm=self.llm, - data=FaithfulnessStatements( - question=question, answer=text, sentences=sentences_with_index - ), + data=StatementGeneratorInput(question=question, answer=text), callbacks=callbacks, ) - - statements = [] - if statements_simplified is None: - return statements - for component in statements_simplified.sentences: - statements.extend(component.simpler_statements) + statements = statements.statements return statements def _compute_score(self, answers: t.Dict) -> float: diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py index daf7b8d03..06e02608a 100644 --- a/src/ragas/metrics/base.py +++ b/src/ragas/metrics/base.py @@ -9,7 +9,6 @@ from enum import Enum from pydantic import ValidationError -from pysbd import Segmenter from tqdm import tqdm from ragas._analytics import EvaluationEvent, _analytics_batcher @@ -19,12 +18,7 @@ from ragas.losses import BinaryMetricLoss, MSELoss from ragas.prompt import FewShotPydanticPrompt, PromptMixin from ragas.run_config import RunConfig -from ragas.utils import ( - RAGAS_SUPPORTED_LANGUAGE_CODES, - camel_to_snake, - deprecated, - get_metric_language, -) +from ragas.utils import camel_to_snake, deprecated, get_metric_language if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks @@ -694,22 +688,4 @@ def from_discrete( return verdict_agg -def get_segmenter( - language: str = "english", clean: bool = False, char_span: bool = False -): - """ - Get a sentence segmenter for a given language - """ - language = language.lower() - if language not in RAGAS_SUPPORTED_LANGUAGE_CODES: - raise ValueError( - f"Language '{language}' not supported. Supported languages: {RAGAS_SUPPORTED_LANGUAGE_CODES.keys()}" - ) - return Segmenter( - language=RAGAS_SUPPORTED_LANGUAGE_CODES[language], - clean=clean, - char_span=char_span, - ) - - ensembler = Ensember() diff --git a/src/ragas/prompt/base.py b/src/ragas/prompt/base.py index 36bd60bd2..6e315a4b5 100644 --- a/src/ragas/prompt/base.py +++ b/src/ragas/prompt/base.py @@ -7,7 +7,7 @@ from langchain_core.prompt_values import StringPromptValue from pydantic import BaseModel -from ragas.utils import RAGAS_SUPPORTED_LANGUAGE_CODES, camel_to_snake +from ragas.utils import camel_to_snake if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks @@ -17,13 +17,6 @@ logger = logging.getLogger(__name__) -def _check_if_language_is_supported(language: str): - if language not in RAGAS_SUPPORTED_LANGUAGE_CODES: - raise ValueError( - f"Language '{language}' not supported. Supported languages: {RAGAS_SUPPORTED_LANGUAGE_CODES.keys()}" - ) - - class BasePrompt(ABC): def __init__( self, @@ -34,7 +27,6 @@ def __init__( if name is None: self.name = camel_to_snake(self.__class__.__name__) - _check_if_language_is_supported(language) self.language = language self.original_hash = original_hash diff --git a/src/ragas/prompt/mixin.py b/src/ragas/prompt/mixin.py index 66ba13740..c354a8d9e 100644 --- a/src/ragas/prompt/mixin.py +++ b/src/ragas/prompt/mixin.py @@ -5,7 +5,6 @@ import os import typing as t -from .base import _check_if_language_is_supported from .pydantic_prompt import PydanticPrompt if t.TYPE_CHECKING: @@ -111,7 +110,6 @@ def load_prompts(self, path: str, language: t.Optional[str] = None): "Language not specified, loading prompts for default language: %s", language, ) - _check_if_language_is_supported(language) loaded_prompts = {} for prompt_name, prompt in self.get_prompts().items(): diff --git a/src/ragas/prompt/pydantic_prompt.py b/src/ragas/prompt/pydantic_prompt.py index 938a53473..fb1408533 100644 --- a/src/ragas/prompt/pydantic_prompt.py +++ b/src/ragas/prompt/pydantic_prompt.py @@ -15,7 +15,7 @@ from ragas.callbacks import ChainType, new_group from ragas.exceptions import RagasOutputParserException -from .base import BasePrompt, StringIO, _check_if_language_is_supported +from .base import BasePrompt, StringIO from .utils import extract_json, get_all_strings, update_strings if t.TYPE_CHECKING: @@ -227,9 +227,6 @@ async def adapt( Adapt the prompt to a new language. """ - # throws ValueError if language is not supported - _check_if_language_is_supported(target_language) - # set the original hash, this is used to # identify the original prompt object when loading from file if self.original_hash is None: diff --git a/src/ragas/utils.py b/src/ragas/utils.py index 06eb2ee36..d761e52c0 100644 --- a/src/ragas/utils.py +++ b/src/ragas/utils.py @@ -11,15 +11,11 @@ import numpy as np import tiktoken from datasets import Dataset -from pysbd.languages import LANGUAGE_CODES if t.TYPE_CHECKING: from ragas.metrics.base import Metric DEBUG_ENV_VAR = "RAGAS_DEBUG" -RAGAS_SUPPORTED_LANGUAGE_CODES = { - v.__name__.lower(): k for k, v in LANGUAGE_CODES.items() -} @lru_cache(maxsize=1)