Skip to content

Commit

Permalink
fix: add reference to simple scoring
Browse files Browse the repository at this point in the history
  • Loading branch information
shahules786 committed Dec 13, 2024
1 parent 14695dd commit 5ebc0e2
Showing 1 changed file with 8 additions and 12 deletions.
20 changes: 8 additions & 12 deletions src/ragas/metrics/_simple_criteria.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,7 @@ class SingleTurnSimpleCriteriaInput(BaseModel):


class MultiTurnSimpleCriteriaInput(BaseModel):
user_input: t.Optional[str] = Field(
description="The input to the model", default=None
)
user_input: str = Field(description="The input to the model")
reference: t.Optional[str] = Field(
description="The reference response", default=None
)
Expand Down Expand Up @@ -172,20 +170,18 @@ async def _single_turn_ascore(
async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
assert self.llm is not None, "set LLM before use"

user_input, context, response = (
row["user_input"],
user_input, response, retrieved_contexts, reference = (
row.get("user_input"),
row.get("response"),
row.get("retrieved_contexts"),
row["response"],
row.get("reference"),
)

if context is not None:
if isinstance(context, list):
context = "\n".join(context)
user_input = f"Question: {user_input} Answer using context: {context}"

prompt_input = SingleTurnSimpleCriteriaInput(
user_input=user_input,
response=response,
retrieved_contexts=retrieved_contexts,
reference=reference,
)

response = await self.single_turn_prompt.generate(
Expand All @@ -200,11 +196,11 @@ async def _multi_turn_ascore(
self, sample: MultiTurnSample, callbacks: Callbacks
) -> float:
assert self.llm is not None, "LLM is not set"
assert sample.reference is not None, "Reference is not set"

interaction = sample.pretty_repr()
prompt_input = MultiTurnSimpleCriteriaInput(
user_input=interaction,
reference=sample.reference,
)
response = await self.multi_turn_prompt.generate(
data=prompt_input,
Expand Down

0 comments on commit 5ebc0e2

Please sign in to comment.