feat: migrate context recall, answer relevancy, and context entity recall metrics to modular BasePrompt architecture (vibrantlabsai#2435)

Author: anistark

src/ragas/metrics/collections/__init__.py

@@ -2,10 +2,7 @@
 
 from ragas.metrics.collections._answer_accuracy import AnswerAccuracy
 from ragas.metrics.collections._answer_correctness import AnswerCorrectness
-from ragas.metrics.collections._answer_relevancy import AnswerRelevancy
 from ragas.metrics.collections._bleu_score import BleuScore
-from ragas.metrics.collections._context_entity_recall import ContextEntityRecall
-from ragas.metrics.collections._context_recall import ContextRecall
 from ragas.metrics.collections._context_relevance import ContextRelevance
 from ragas.metrics.collections._factual_correctness import FactualCorrectness
 from ragas.metrics.collections._faithfulness import Faithfulness
@@ -20,13 +17,16 @@
     StringPresence,
 )
 from ragas.metrics.collections._summary_score import SummaryScore
+from ragas.metrics.collections.answer_relevancy import AnswerRelevancy
 from ragas.metrics.collections.base import BaseMetric
+from ragas.metrics.collections.context_entity_recall import ContextEntityRecall
 from ragas.metrics.collections.context_precision import (
     ContextPrecision,
     ContextPrecisionWithoutReference,
     ContextPrecisionWithReference,
     ContextUtilization,
 )
+from ragas.metrics.collections.context_recall import ContextRecall
 
 __all__ = [
     "BaseMetric",  # Base class

src/ragas/metrics/collections/answer_relevancy/__init__.py

@@ -0,0 +1,7 @@
+"""Answer Relevancy metrics v2 - Modern implementation."""
+
+from .metric import AnswerRelevancy
+
+__all__ = [
+    "AnswerRelevancy",
+]

…metrics/collections/_answer_relevancy.py …s/collections/answer_relevancy/metric.pysrc/ragas/metrics/collections/_answer_relevancy.py renamed to src/ragas/metrics/collections/answer_relevancy/metric.py src/ragas/metrics/collections/_answer_relevancy.py renamed to src/ragas/metrics/collections/answer_relevancy/metric.py

@@ -1,43 +1,45 @@
-"""Answer Relevancy metric v2 - Class-based implementation with modern components."""
+"""Answer Relevancy metrics v2 - Modern implementation with structured prompts."""
 
 import typing as t
 
 import numpy as np
-from pydantic import BaseModel
 
 from ragas.metrics.collections.base import BaseMetric
 from ragas.metrics.result import MetricResult
-from ragas.prompt.metrics.answer_relevance import answer_relevancy_prompt
+
+from .util import (
+    AnswerRelevanceInput,
+    AnswerRelevanceOutput,
+    AnswerRelevancePrompt,
+)
 
 if t.TYPE_CHECKING:
     from ragas.embeddings.base import BaseRagasEmbedding
     from ragas.llms.base import InstructorBaseRagasLLM
 
 
-class AnswerRelevanceOutput(BaseModel):
-    """Structured output for answer relevance question generation."""
-
-    question: str
-    noncommittal: int
-
-
 class AnswerRelevancy(BaseMetric):
     """
-    Evaluate answer relevancy by generating questions from the response and comparing to original question.
+    Modern v2 implementation of answer relevancy evaluation.
+
+    Evaluates answer relevancy by generating multiple questions from the response
+    and comparing them to the original question using cosine similarity.
+    The metric detects evasive/noncommittal answers.
 
-    This implementation uses modern instructor LLMs with structured output and modern embeddings.
+    This implementation uses modern instructor LLMs with structured output
+    and modern embeddings for semantic comparison.
     Only supports modern components - legacy wrappers are rejected with clear error messages.
 
     Usage:
-        >>> from openai import AsyncOpenAI
-        >>> from ragas.llms import llm_factory
+        >>> import openai
+        >>> from ragas.llms.base import llm_factory
         >>> from ragas.embeddings.base import embedding_factory
         >>> from ragas.metrics.collections import AnswerRelevancy
         >>>
         >>> # Setup dependencies
-        >>> client = AsyncOpenAI()
+        >>> client = openai.AsyncOpenAI()
         >>> llm = llm_factory("gpt-4o-mini", client=client)
-        >>> embeddings = embedding_factory("openai", model="text-embedding-ada-002", client=client, interface="modern")
+        >>> embeddings = embedding_factory("openai", model="text-embedding-ada-002", client=client)
         >>>
         >>> # Create metric instance
         >>> metric = AnswerRelevancy(llm=llm, embeddings=embeddings, strictness=3)
@@ -47,20 +49,14 @@ class AnswerRelevancy(BaseMetric):
         ...     user_input="What is the capital of France?",
         ...     response="Paris is the capital of France."
         ... )
-        >>> print(f"Score: {result.value}")
-        >>>
-        >>> # Batch evaluation
-        >>> results = await metric.abatch_score([
-        ...     {"user_input": "Q1", "response": "A1"},
-        ...     {"user_input": "Q2", "response": "A2"},
-        ... ])
+        >>> print(f"Answer Relevancy: {result.value}")
 
     Attributes:
         llm: Modern instructor-based LLM for question generation
-        embeddings: Modern embeddings model with embed_text() and embed_texts() methods
+        embeddings: Modern embeddings model for semantic comparison
         name: The metric name
-        strictness: Number of questions to generate per answer (3-5 recommended)
-        allowed_values: Score range (0.0 to 1.0)
+        strictness: Number of questions to generate (default: 3)
+        allowed_values: Score range (0.0 to 1.0, higher is better)
     """
 
     # Type hints for linter (attributes are set in __init__)
@@ -75,13 +71,23 @@ def __init__(
         strictness: int = 3,
         **kwargs,
     ):
-        """Initialize AnswerRelevancy metric with required components."""
+        """
+        Initialize AnswerRelevancy metric with required components.
+
+        Args:
+            llm: Modern instructor-based LLM for question generation
+            embeddings: Modern embeddings model for semantic comparison
+            name: The metric name (default: "answer_relevancy")
+            strictness: Number of questions to generate (default: 3)
+            **kwargs: Additional arguments passed to BaseMetric
+        """
         # Set attributes explicitly before calling super()
         self.llm = llm
         self.embeddings = embeddings
         self.strictness = strictness
+        self.prompt = AnswerRelevancePrompt()  # Initialize prompt class once
 
-        # Call super() for validation (without passing llm/embeddings in kwargs)
+        # Call super() for validation
         super().__init__(name=name, **kwargs)
 
     async def ascore(self, user_input: str, response: str) -> MetricResult:
@@ -95,15 +101,23 @@ async def ascore(self, user_input: str, response: str) -> MetricResult:
             response: The response to evaluate
 
         Returns:
-            MetricResult with relevancy score (0.0-1.0)
+            MetricResult with relevancy score (0.0-1.0, higher is better)
         """
-        prompt = answer_relevancy_prompt(response)
+        # Input validation
+        if not user_input:
+            raise ValueError("user_input cannot be empty")
+        if not response:
+            raise ValueError("response cannot be empty")
 
+        # Generate multiple questions from response
         generated_questions = []
         noncommittal_flags = []
 
         for _ in range(self.strictness):
-            result = await self.llm.agenerate(prompt, AnswerRelevanceOutput)
+            # Create input data and generate prompt
+            input_data = AnswerRelevanceInput(response=response)
+            prompt_string = self.prompt.to_string(input_data)
+            result = await self.llm.agenerate(prompt_string, AnswerRelevanceOutput)
 
             if result.question:
                 generated_questions.append(result.question)
@@ -112,13 +126,18 @@ async def ascore(self, user_input: str, response: str) -> MetricResult:
         if not generated_questions:
             return MetricResult(value=0.0)
 
+        # Check if all responses are noncommittal
         all_noncommittal = np.all(noncommittal_flags)
 
+        # Embed the original question
         question_vec = np.asarray(self.embeddings.embed_text(user_input)).reshape(1, -1)
+
+        # Embed the generated questions
         gen_question_vec = np.asarray(
             self.embeddings.embed_texts(generated_questions)
         ).reshape(len(generated_questions), -1)
 
+        # Calculate cosine similarity
         norm = np.linalg.norm(gen_question_vec, axis=1) * np.linalg.norm(
             question_vec, axis=1
         )
@@ -129,6 +148,7 @@ async def ascore(self, user_input: str, response: str) -> MetricResult:
             / norm
         )
 
+        # Score is average cosine similarity, reduced to 0 if response is noncommittal
         score = cosine_sim.mean() * int(not all_noncommittal)
 
         return MetricResult(value=float(score))

src/ragas/metrics/collections/answer_relevancy/util.py

@@ -0,0 +1,64 @@
+"""Answer Relevancy prompt classes and models."""
+
+from pydantic import BaseModel, Field
+
+from ragas.prompt.metrics.base_prompt import BasePrompt
+
+
+class AnswerRelevanceInput(BaseModel):
+    """Input model for answer relevance evaluation."""
+
+    response: str = Field(
+        ..., description="The response/answer to generate questions from"
+    )
+
+
+class AnswerRelevanceOutput(BaseModel):
+    """Structured output for answer relevance question generation."""
+
+    question: str = Field(
+        ..., description="Question that can be answered from the response"
+    )
+    noncommittal: int = Field(
+        ...,
+        description="1 if the response is evasive/vague, 0 if it is substantive",
+    )
+
+
+class AnswerRelevancePrompt(BasePrompt[AnswerRelevanceInput, AnswerRelevanceOutput]):
+    """Answer relevance evaluation prompt with structured input/output."""
+
+    input_model = AnswerRelevanceInput
+    output_model = AnswerRelevanceOutput
+
+    instruction = """Generate a question for the given answer and identify if the answer is noncommittal.
+Give noncommittal as 1 if the answer is noncommittal (evasive, vague, or ambiguous) and 0 if the answer is substantive.
+Examples of noncommittal answers: "I don't know", "I'm not sure", "It depends"."""
+
+    examples = [
+        (
+            AnswerRelevanceInput(response="Albert Einstein was born in Germany."),
+            AnswerRelevanceOutput(
+                question="Where was Albert Einstein born?",
+                noncommittal=0,
+            ),
+        ),
+        (
+            AnswerRelevanceInput(
+                response="The capital of France is Paris, a city known for its architecture and culture."
+            ),
+            AnswerRelevanceOutput(
+                question="What is the capital of France?",
+                noncommittal=0,
+            ),
+        ),
+        (
+            AnswerRelevanceInput(
+                response="I don't know about the groundbreaking feature of the smartphone invented in 2023 as I am unaware of information beyond 2022."
+            ),
+            AnswerRelevanceOutput(
+                question="What was the groundbreaking feature of the smartphone invented in 2023?",
+                noncommittal=1,
+            ),
+        ),
+    ]

src/ragas/metrics/collections/context_entity_recall/__init__.py

@@ -0,0 +1,7 @@
+"""Context Entity Recall metrics v2 - Modern implementation."""
+
+from .metric import ContextEntityRecall
+
+__all__ = [
+    "ContextEntityRecall",
+]