Skip to content

Commit

Permalink
Provide more deepeval metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
penguine-ip authored Mar 9, 2024
1 parent 5e1d8b0 commit 39a7449
Showing 1 changed file with 32 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
ContextualRecallMetric, # type: ignore
ContextualRelevancyMetric, # type: ignore
FaithfulnessMetric, # type: ignore
BiasMetric, # type: ignore
ToxicityMetric, # type: ignore
)
from deepeval.test_case import LLMTestCase

Expand Down Expand Up @@ -48,6 +50,14 @@ class DeepEvalMetric(Enum):
#: Inputs - `questions: List[str], contexts: List[List[str]], responses: List[str]`
CONTEXTUAL_RELEVANCE = "contextual_relevance"

#: Bias.\
#: Inputs - `questions: List[str], responses: List[str]`
BIAS = "bias"

#: Toxicity.\
#: Inputs - `questions: List[str], responses: List[str]`
TOXICITY = "bias"

def __str__(self):
return self.value

Expand Down Expand Up @@ -180,7 +190,16 @@ def validate_input_parameters(metric: DeepEvalMetric, expected: Dict[str, Any],
if param not in received:
msg = f"DeepEval evaluator expected input parameter '{param}' for metric '{metric}'"
raise ValueError(msg)


@staticmethod
def question_response(
questions: List[str], responses: List[str]
) -> Iterable[LLMTestCase]:
InputConverters._validate_input_elements(questions=questions, responses=responses)
for q, r in zip(questions, responses): # type: ignore
test_case = LLMTestCase(input=q, actual_output=r)
yield test_case

@staticmethod
def question_context_response(
questions: List[str], contexts: List[List[str]], responses: List[str]
Expand Down Expand Up @@ -255,4 +274,16 @@ def inner(output: TestResult, metric: DeepEvalMetric) -> List[MetricResult]:
InputConverters.question_context_response, # type: ignore
init_parameters={"model": Optional[str]}, # type: ignore
),
DeepEvalMetric.BIAS: MetricDescriptor.new(
DeepEvalMetric.BIAS,
BiasMetric,
InputConverters.question_response, # type: ignore
init_parameters={"model": Optional[str]}, # type: ignore
),
DeepEvalMetric.TOXICITY: MetricDescriptor.new(
DeepEvalMetric.TOXICITY,
ToxicityMetric,
InputConverters.question_response, # type: ignore
init_parameters={"model": Optional[str]}, # type: ignore
),
}

0 comments on commit 39a7449

Please sign in to comment.