code: """Relevancy evaluation."""
  from __future__ import annotations
  
  import asyncio
  from typing import Any, Optional, Sequence, Union
  
  from llama_index.core import ServiceContext
  from llama_index.core.evaluation.base import BaseEvaluator, EvaluationResult
  from llama_index.core.indices import SummaryIndex
  from llama_index.core.llms.llm import LLM
  from llama_index.core.prompts import BasePromptTemplate, PromptTemplate
  from llama_index.core.prompts.mixin import PromptDictType
  from llama_index.core.schema import Document
  from llama_index.core.settings import Settings, llm_from_settings_or_context
  
  DEFAULT_EVAL_TEMPLATE = PromptTemplate(
      "Your task is to evaluate if the response for the query \
      is in line with the context information provided.\n"
      "You have two options to answer. Either YES/ NO.\n"
      "Answer - YES, if the response for the query \
      is in line with context information otherwise NO.\n"
      "Query and Response: \n {query_str}\n"
      "Context: \n {context_str}\n"
      "Answer: "
  )
  
  DEFAULT_REFINE_TEMPLATE = PromptTemplate(
      "We want to understand if the following query and response is"
      "in line with the context information: \n {query_str}\n"
      "We have provided an existing YES/NO answer: \n {existing_answer}\n"
      "We have the opportunity to refine the existing answer "
      "(only if needed) with some more context below.\n"
      "------------\n"
      "{context_msg}\n"
      "------------\n"
      "If the existing answer was already YES, still answer YES. "
      "If the information is present in the new context, answer YES. "
      "Otherwise answer NO.\n"
  )
  
  
  class RelevancyEvaluator(BaseEvaluator):
      """Relenvancy evaluator.
  
      Evaluates the relevancy of retrieved contexts and response to a query.
      This evaluator considers the query string, retrieved contexts, and response string.
  
      Args:
          service_context(Optional[ServiceContext]):
              The service context to use for evaluation.
          raise_error(Optional[bool]):
              Whether to raise an error if the response is invalid.
              Defaults to False.
          eval_template(Optional[Union[str, BasePromptTemplate]]):
              The template to use for evaluation.
          refine_template(Optional[Union[str, BasePromptTemplate]]):
              The template to use for refinement.
      """
  
      def __init__(
          self,
          llm: Optional[LLM] = None,
          raise_error: bool = False,
          eval_template: Optional[Union[str, BasePromptTemplate]] = None,
          refine_template: Optional[Union[str, BasePromptTemplate]] = None,
          # deprecated
          service_context: Optional[ServiceContext] = None,
      ) -> None:
          """Init params."""
          self._llm = llm or llm_from_settings_or_context(Settings, service_context)
          self._raise_error = raise_error
  
          self._eval_template: BasePromptTemplate
          if isinstance(eval_template, str):
              self._eval_template = PromptTemplate(eval_template)
          else:
              self._eval_template = eval_template or DEFAULT_EVAL_TEMPLATE
  
          self._refine_template: BasePromptTemplate
          if isinstance(refine_template, str):
              self._refine_template = PromptTemplate(refine_template)
          else:
              self._refine_template = refine_template or DEFAULT_REFINE_TEMPLATE
  
      def _get_prompts(self) -> PromptDictType:
          """Get prompts."""
          return {
              "eval_template": self._eval_template,
              "refine_template": self._refine_template,
          }
  
      def _update_prompts(self, prompts: PromptDictType) -> None:
          """Update prompts."""
          if "eval_template" in prompts:
              self._eval_template = prompts["eval_template"]
          if "refine_template" in prompts:
              self._refine_template = prompts["refine_template"]
  
      async def aevaluate(
          self,
          query: str | None = None,
          response: str | None = None,
          contexts: Sequence[str] | None = None,
          sleep_time_in_seconds: int = 0,
          **kwargs: Any,
      ) -> EvaluationResult:
          """Evaluate whether the contexts and response are relevant to the query."""
          del kwargs  # Unused
  
          if query is None or contexts is None or response is None:
              raise ValueError("query, contexts, and response must be provided")
  
          docs = [Document(text=context) for context in contexts]
          index = SummaryIndex.from_documents(docs)
  
          query_response = f"Question: {query}\nResponse: {response}"
  
          await asyncio.sleep(sleep_time_in_seconds)
  
          query_engine = index.as_query_engine(
              llm=self._llm,
              text_qa_template=self._eval_template,
              refine_template=self._refine_template,
          )
          response_obj = await query_engine.aquery(query_response)
  
          raw_response_txt = str(response_obj)
  
          if "yes" in raw_response_txt.lower():
              passing = True
          else:
              if self._raise_error:
                  raise ValueError("The response is invalid")
              passing = False
  
          return EvaluationResult(
              query=query,
              response=response,
              passing=passing,
              score=1.0 if passing else 0.0,
              feedback=raw_response_txt,
              contexts=contexts,
          )
  
  
  QueryResponseEvaluator = RelevancyEvaluator
  
apis: llama_index.core.prompts.PromptTemplate llama_index.core.settings.llm_from_settings_or_context llama_index.core.evaluation.base.EvaluationResult llama_index.core.schema.Document llama_index.core.indices.SummaryIndex.from_documents
extract_api: [((620, 974), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['"""Your task is to evaluate if the response for the query     is in line with the context information provided.\nYou have two options to answer. Either YES/ NO.\nAnswer - YES, if the response for the query     is in line with context information otherwise NO.\nQuery and Response: \n {query_str}\nContext: \n {context_str}\nAnswer: """'], {}), '(\n    """Your task is to evaluate if the response for the query     is in line with the context information provided.\nYou have two options to answer. Either YES/ NO.\nAnswer - YES, if the response for the query     is in line with context information otherwise NO.\nQuery and Response: \n {query_str}\nContext: \n {context_str}\nAnswer: """\n    )\n', (634, 974), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((1040, 1530), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['"""We want to understand if the following query and response isin line with the context information: \n {query_str}\nWe have provided an existing YES/NO answer: \n {existing_answer}\nWe have the opportunity to refine the existing answer (only if needed) with some more context below.\n------------\n{context_msg}\n------------\nIf the existing answer was already YES, still answer YES. If the information is present in the new context, answer YES. Otherwise answer NO.\n"""'], {}), '(\n    """We want to understand if the following query and response isin line with the context information: \n {query_str}\nWe have provided an existing YES/NO answer: \n {existing_answer}\nWe have the opportunity to refine the existing answer (only if needed) with some more context below.\n------------\n{context_msg}\n------------\nIf the existing answer was already YES, still answer YES. If the information is present in the new context, answer YES. Otherwise answer NO.\n"""\n    )\n', (1054, 1530), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((4408, 4441), 'llama_index.core.indices.SummaryIndex.from_documents', 'SummaryIndex.from_documents', (['docs'], {}), '(docs)\n', (4435, 4441), False, 'from llama_index.core.indices import SummaryIndex\n'), ((5085, 5231), 'llama_index.core.evaluation.base.EvaluationResult', 'EvaluationResult', ([], {'query': 'query', 'response': 'response', 'passing': 'passing', 'score': '(1.0 if passing else 0.0)', 'feedback': 'raw_response_txt', 'contexts': 'contexts'}), '(query=query, response=response, passing=passing, score=1.0 if\n    passing else 0.0, feedback=raw_response_txt, contexts=contexts)\n', (5101, 5231), False, 'from llama_index.core.evaluation.base import BaseEvaluator, EvaluationResult\n'), ((2722, 2777), 'llama_index.core.settings.llm_from_settings_or_context', 'llm_from_settings_or_context', (['Settings', 'service_context'], {}), '(Settings, service_context)\n', (2750, 2777), False, 'from llama_index.core.settings import Settings, llm_from_settings_or_context\n'), ((2944, 2973), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['eval_template'], {}), '(eval_template)\n', (2958, 2973), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((3193, 3224), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['refine_template'], {}), '(refine_template)\n', (3207, 3224), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((4344, 4366), 'llama_index.core.schema.Document', 'Document', ([], {'text': 'context'}), '(text=context)\n', (4352, 4366), False, 'from llama_index.core.schema import Document\n'), ((4526, 4562), 'asyncio.sleep', 'asyncio.sleep', (['sleep_time_in_seconds'], {}), '(sleep_time_in_seconds)\n', (4539, 4562), False, 'import asyncio\n')]