code: """Faithfulness evaluation."""
  
  from __future__ import annotations
  
  from typing import Any, List, Optional, Sequence, Union
  
  from llama_index.core.evaluation.base import BaseEvaluator, EvaluationResult
  from llama_index.core.multi_modal_llms.base import MultiModalLLM
  from llama_index.core.prompts import BasePromptTemplate, PromptTemplate
  from llama_index.core.prompts.mixin import PromptDictType
  from llama_index.core.schema import ImageNode
  
  DEFAULT_EVAL_TEMPLATE = PromptTemplate(
      "Please tell if a given piece of information "
      "is supported by the visual as well as textual context information.\n"
      "You need to answer with either YES or NO.\n"
      "Answer YES if any of the image(s) and textual context supports the information, even "
      "if most of the context is unrelated. "
      "Some examples are provided below with only text context, but please do use\n"
      "any images for context if they are provided.\n\n"
      "Information: Apple pie is generally double-crusted.\n"
      "Context: An apple pie is a fruit pie in which the principal filling "
      "ingredient is apples. \n"
      "Apple pie is often served with whipped cream, ice cream "
      "('apple pie à la mode'), custard or cheddar cheese.\n"
      "It is generally double-crusted, with pastry both above "
      "and below the filling; the upper crust may be solid or "
      "latticed (woven of crosswise strips).\n"
      "Answer: YES\n"
      "Information: Apple pies tastes bad.\n"
      "Context: An apple pie is a fruit pie in which the principal filling "
      "ingredient is apples. \n"
      "Apple pie is often served with whipped cream, ice cream "
      "('apple pie à la mode'), custard or cheddar cheese.\n"
      "It is generally double-crusted, with pastry both above "
      "and below the filling; the upper crust may be solid or "
      "latticed (woven of crosswise strips).\n"
      "Answer: NO\n"
      "Information: {query_str}\n"
      "Context: {context_str}\n"
      "Answer: "
  )
  
  DEFAULT_REFINE_TEMPLATE = PromptTemplate(
      "We want to understand if the following information is present "
      "in the context information: {query_str}\n"
      "We have provided an existing YES/NO answer: {existing_answer}\n"
      "We have the opportunity to refine the existing answer "
      "(only if needed) with some more context below.\n"
      "------------\n"
      "{context_msg}\n"
      "------------\n"
      "If the existing answer was already YES, still answer YES. "
      "If the information is present in the new context, answer YES. "
      "Otherwise answer NO.\n"
  )
  
  
  class MultiModalFaithfulnessEvaluator(BaseEvaluator):
      """Multi-Modal Faithfulness evaluator.
  
      Evaluates whether a response is faithful to the contexts
      (i.e. whether the response is supported by the contexts or hallucinated.)
  
      This evaluator only considers the response string and the list of context strings.
  
      Args:
          multi_modal_llm(Optional[MultiModalLLM]):
              The Multi-Modal LLM Judge to use for evaluations.
          raise_error(bool): Whether to raise an error when the response is invalid.
              Defaults to False.
          eval_template(Optional[Union[str, BasePromptTemplate]]):
              The template to use for evaluation.
          refine_template(Optional[Union[str, BasePromptTemplate]]):
              The template to use for refining the evaluation.
      """
  
      def __init__(
          self,
          multi_modal_llm: Optional[MultiModalLLM] = None,
          raise_error: bool = False,
          eval_template: Union[str, BasePromptTemplate, None] = None,
          refine_template: Union[str, BasePromptTemplate, None] = None,
      ) -> None:
          """Init params."""
          if multi_modal_llm is None:
              try:
                  from llama_index.multi_modal_llms.openai import (
                      OpenAIMultiModal,
                  )  # pants: no-infer-dep
              except ImportError:
                  raise ImportError(
                      "OpenAIMultiModal is not installed. "
                      "Please install it using `pip install llama-index-multi-modal-llms-openai`"
                  )
  
              self._multi_modal_llm = OpenAIMultiModal(
                  model="gpt-4-vision-preview", max_new_tokens=1000
              )
          else:
              self._multi_modal_llm = multi_modal_llm
  
          self._raise_error = raise_error
  
          self._eval_template: BasePromptTemplate
          if isinstance(eval_template, str):
              self._eval_template = PromptTemplate(eval_template)
          else:
              self._eval_template = eval_template or DEFAULT_EVAL_TEMPLATE
  
          self._refine_template: BasePromptTemplate
          if isinstance(refine_template, str):
              self._refine_template = PromptTemplate(refine_template)
          else:
              self._refine_template = refine_template or DEFAULT_REFINE_TEMPLATE
  
      def _get_prompts(self) -> PromptDictType:
          """Get prompts."""
          return {
              "eval_template": self._eval_template,
              "refine_template": self._refine_template,
          }
  
      def _update_prompts(self, prompts: PromptDictType) -> None:
          """Update prompts."""
          if "eval_template" in prompts:
              self._eval_template = prompts["eval_template"]
          if "refine_template" in prompts:
              self._refine_template = prompts["refine_template"]
  
      def evaluate(
          self,
          query: Union[str, None] = None,
          response: Union[str, None] = None,
          contexts: Union[Sequence[str], None] = None,
          image_paths: Union[List[str], None] = None,
          image_urls: Union[List[str], None] = None,
          **kwargs: Any,
      ) -> EvaluationResult:
          """Evaluate whether the response is faithful to the multi-modal contexts."""
          del query  # Unused
          del kwargs  # Unused
          if contexts is None or response is None:
              raise ValueError("contexts and response must be provided")
  
          context_str = "\n\n".join(contexts)
          fmt_prompt = self._eval_template.format(
              context_str=context_str, query_str=response
          )
  
          if image_paths:
              image_nodes = [
                  ImageNode(image_path=image_path) for image_path in image_paths
              ]
          if image_urls:
              image_nodes = [ImageNode(image_url=image_url) for image_url in image_urls]
  
          response_obj = self._multi_modal_llm.complete(
              prompt=fmt_prompt,
              image_documents=image_nodes,
          )
  
          raw_response_txt = str(response_obj)
  
          if "yes" in raw_response_txt.lower():
              passing = True
          else:
              passing = False
              if self._raise_error:
                  raise ValueError("The response is invalid")
  
          return EvaluationResult(
              response=response,
              contexts=contexts,
              passing=passing,
              score=1.0 if passing else 0.0,
              feedback=raw_response_txt,
          )
  
      async def aevaluate(
          self,
          query: Union[str, None] = None,
          response: Union[str, None] = None,
          contexts: Union[Sequence[str], None] = None,
          image_paths: Union[List[str], None] = None,
          image_urls: Union[List[str], None] = None,
          **kwargs: Any,
      ) -> EvaluationResult:
          """Async evaluate whether the response is faithful to the multi-modal contexts."""
          del query  # Unused
          del kwargs  # Unused
          if contexts is None or response is None:
              raise ValueError("contexts and response must be provided")
  
          context_str = "\n\n".join(contexts)
          fmt_prompt = self._eval_template.format(
              context_str=context_str, query_str=response
          )
  
          if image_paths:
              image_nodes = [
                  ImageNode(image_path=image_path) for image_path in image_paths
              ]
          if image_urls:
              image_nodes = [ImageNode(image_url=image_url) for image_url in image_urls]
  
          response_obj = await self._multi_modal_llm.acomplete(
              prompt=fmt_prompt,
              image_documents=image_nodes,
          )
  
          raw_response_txt = str(response_obj)
  
          if "yes" in raw_response_txt.lower():
              passing = True
          else:
              passing = False
              if self._raise_error:
                  raise ValueError("The response is invalid")
  
          return EvaluationResult(
              response=response,
              contexts=contexts,
              passing=passing,
              score=1.0 if passing else 0.0,
              feedback=raw_response_txt,
          )
  
apis: llama_index.core.prompts.PromptTemplate llama_index.multi_modal_llms.openai.OpenAIMultiModal llama_index.core.evaluation.base.EvaluationResult llama_index.core.schema.ImageNode
extract_api: [((468, 1757), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['"""Please tell if a given piece of information is supported by the visual as well as textual context information.\nYou need to answer with either YES or NO.\nAnswer YES if any of the image(s) and textual context supports the information, even if most of the context is unrelated. Some examples are provided below with only text context, but please do use\nany images for context if they are provided.\n\nInformation: Apple pie is generally double-crusted.\nContext: An apple pie is a fruit pie in which the principal filling ingredient is apples. \nApple pie is often served with whipped cream, ice cream (\'apple pie à la mode\'), custard or cheddar cheese.\nIt is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).\nAnswer: YES\nInformation: Apple pies tastes bad.\nContext: An apple pie is a fruit pie in which the principal filling ingredient is apples. \nApple pie is often served with whipped cream, ice cream (\'apple pie à la mode\'), custard or cheddar cheese.\nIt is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).\nAnswer: NO\nInformation: {query_str}\nContext: {context_str}\nAnswer: """'], {}), '(\n    """Please tell if a given piece of information is supported by the visual as well as textual context information.\nYou need to answer with either YES or NO.\nAnswer YES if any of the image(s) and textual context supports the information, even if most of the context is unrelated. Some examples are provided below with only text context, but please do use\nany images for context if they are provided.\n\nInformation: Apple pie is generally double-crusted.\nContext: An apple pie is a fruit pie in which the principal filling ingredient is apples. \nApple pie is often served with whipped cream, ice cream (\'apple pie à la mode\'), custard or cheddar cheese.\nIt is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).\nAnswer: YES\nInformation: Apple pies tastes bad.\nContext: An apple pie is a fruit pie in which the principal filling ingredient is apples. \nApple pie is often served with whipped cream, ice cream (\'apple pie à la mode\'), custard or cheddar cheese.\nIt is generally double-crusted, with pastry both above and below the filling; the upper crust may be solid or latticed (woven of crosswise strips).\nAnswer: NO\nInformation: {query_str}\nContext: {context_str}\nAnswer: """\n    )\n', (482, 1757), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((1983, 2461), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['"""We want to understand if the following information is present in the context information: {query_str}\nWe have provided an existing YES/NO answer: {existing_answer}\nWe have the opportunity to refine the existing answer (only if needed) with some more context below.\n------------\n{context_msg}\n------------\nIf the existing answer was already YES, still answer YES. If the information is present in the new context, answer YES. Otherwise answer NO.\n"""'], {}), '(\n    """We want to understand if the following information is present in the context information: {query_str}\nWe have provided an existing YES/NO answer: {existing_answer}\nWe have the opportunity to refine the existing answer (only if needed) with some more context below.\n------------\n{context_msg}\n------------\nIf the existing answer was already YES, still answer YES. If the information is present in the new context, answer YES. Otherwise answer NO.\n"""\n    )\n', (1997, 2461), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((6740, 6873), 'llama_index.core.evaluation.base.EvaluationResult', 'EvaluationResult', ([], {'response': 'response', 'contexts': 'contexts', 'passing': 'passing', 'score': '(1.0 if passing else 0.0)', 'feedback': 'raw_response_txt'}), '(response=response, contexts=contexts, passing=passing,\n    score=1.0 if passing else 0.0, feedback=raw_response_txt)\n', (6756, 6873), False, 'from llama_index.core.evaluation.base import BaseEvaluator, EvaluationResult\n'), ((8371, 8504), 'llama_index.core.evaluation.base.EvaluationResult', 'EvaluationResult', ([], {'response': 'response', 'contexts': 'contexts', 'passing': 'passing', 'score': '(1.0 if passing else 0.0)', 'feedback': 'raw_response_txt'}), '(response=response, contexts=contexts, passing=passing,\n    score=1.0 if passing else 0.0, feedback=raw_response_txt)\n', (8387, 8504), False, 'from llama_index.core.evaluation.base import BaseEvaluator, EvaluationResult\n'), ((4123, 4190), 'llama_index.multi_modal_llms.openai.OpenAIMultiModal', 'OpenAIMultiModal', ([], {'model': '"""gpt-4-vision-preview"""', 'max_new_tokens': '(1000)'}), "(model='gpt-4-vision-preview', max_new_tokens=1000)\n", (4139, 4190), False, 'from llama_index.multi_modal_llms.openai import OpenAIMultiModal\n'), ((4454, 4483), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['eval_template'], {}), '(eval_template)\n', (4468, 4483), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((4703, 4734), 'llama_index.core.prompts.PromptTemplate', 'PromptTemplate', (['refine_template'], {}), '(refine_template)\n', (4717, 4734), False, 'from llama_index.core.prompts import BasePromptTemplate, PromptTemplate\n'), ((6143, 6175), 'llama_index.core.schema.ImageNode', 'ImageNode', ([], {'image_path': 'image_path'}), '(image_path=image_path)\n', (6152, 6175), False, 'from llama_index.core.schema import ImageNode\n'), ((6270, 6300), 'llama_index.core.schema.ImageNode', 'ImageNode', ([], {'image_url': 'image_url'}), '(image_url=image_url)\n', (6279, 6300), False, 'from llama_index.core.schema import ImageNode\n'), ((7767, 7799), 'llama_index.core.schema.ImageNode', 'ImageNode', ([], {'image_path': 'image_path'}), '(image_path=image_path)\n', (7776, 7799), False, 'from llama_index.core.schema import ImageNode\n'), ((7894, 7924), 'llama_index.core.schema.ImageNode', 'ImageNode', ([], {'image_url': 'image_url'}), '(image_url=image_url)\n', (7903, 7924), False, 'from llama_index.core.schema import ImageNode\n')]