code: import json
  from typing import Any, Hashable
  
  import nest_asyncio
  import pandas as pd
  import requests
  from llama_index import ServiceContext
  from llama_index.llms import OpenAI
  from ragas import metrics
  from tenacity import retry, stop_after_attempt, wait_random_exponential
  from tqdm import tqdm
  
  from wandbot.evaluation.eval.correctness import (
      CORRECTNESS_EVAL_TEMPLATE,
      WandbCorrectnessEvaluator,
  )
  from wandbot.evaluation.eval.factfulness import (
      FACTFULNESS_EVAL_TEMPLATE,
      WandbFactfulnessEvaluator,
  )
  from wandbot.evaluation.eval.relevancy import (
      RELEVANCY_EVAL_TEMPLATE,
      WandbRelevancyEvaluator,
  )
  from wandbot.utils import cachew, get_logger
  
  logger = get_logger(__name__)
  
  nest_asyncio.apply()
  
  
  EVAL_CACHE = "data/cache/eval_cache/cache.db"
  service_context = ServiceContext.from_defaults(llm=OpenAI("gpt-4-1106-preview"))
  correctness_evaluator = WandbCorrectnessEvaluator(
      service_context=service_context,
      eval_template=CORRECTNESS_EVAL_TEMPLATE,
  )
  faithfulness_evaluator = WandbFactfulnessEvaluator(
      service_context=service_context,
      eval_template=FACTFULNESS_EVAL_TEMPLATE,
  )
  relevancy_evaluator = WandbRelevancyEvaluator(
      service_context=service_context,
      eval_template=RELEVANCY_EVAL_TEMPLATE,
  )
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_answer(question: str, application: str = "api-eval-bharat") -> str:
      url = "http://0.0.0.0:8000/query"
      payload = {
          "question": question,
          "language": "en",
          "application": application,
      }
      response = requests.post(url, data=json.dumps(payload))
      response = response.json()
      return json.dumps(response)
  
  
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_eval_record(row_str: str, application: str = "api-eval-bharat") -> str:
      row = json.loads(row_str)
      response = get_answer(row["question"], application=application)
      response = json.loads(response)
      response["ground_truths"] = row["answer"]
      response["reference_notes"] = row["notes"]
      response["contexts"] = [
          "\nSource: " + source["source"] + " \n " + source["text"]
          for source in json.loads(response["source_documents"])
      ]
      response = json.dumps(response)
      return response
  
  
  def parse_answer_eval(metric: str, row: dict[str, Any]) -> dict[str, Any]:
      result = {
          f"{metric}_score": row.get("score"),
          f"{metric}_result": row.get("passing"),
          f"{metric}_reason": row.get("feedback"),
      }
      return result
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_answer_correctness(row_str: str) -> str:
      row = json.loads(row_str)
      result = correctness_evaluator.evaluate(
          query=row["question"],
          response=row["answer"],
          reference=row["ground_truths"],
          contexts=row["contexts"],
          reference_notes=row["reference_notes"],
      )
      result = parse_answer_eval("answer_correctness", result.dict())
      result[
          "answer_correctness_score_(ragas)"
      ] = metrics.answer_correctness.score_single(row)
      result = json.dumps(result)
      return result
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_answer_relevancy(row_str: str) -> str:
      row = json.loads(row_str)
      result = relevancy_evaluator.evaluate(
          query=row["question"],
          response=row["answer"],
          contexts=row["contexts"],
          reference=row["ground_truths"],
      )
      result = parse_answer_eval("answer_relevancy", result.dict())
      result[
          "answer_relevancy_score_(ragas)"
      ] = metrics.answer_relevancy.score_single(row)
      result = json.dumps(result)
      return result
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_answer_faithfulness(row_str: str) -> str:
      row = json.loads(row_str)
      result = faithfulness_evaluator.evaluate(
          query=row["question"],
          response=row["answer"],
          contexts=row["contexts"],
          reference=row["ground_truths"],
      )
  
      result = parse_answer_eval("answer_faithfulness", result.dict())
      result[
          "answer_faithfulness_score_(ragas)"
      ] = metrics.faithfulness.score_single(row)
      result = json.dumps(result)
      return result
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_answer_similarity(row_str: str) -> str:
      row = json.loads(row_str)
      result = metrics.answer_similarity.score_single(row)
      result = json.dumps({"answer_similarity_score_(ragas)": result})
      return result
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_context_precision(row_str: str) -> str:
      row = json.loads(row_str)
      result = metrics.context_precision.score_single(row)
      result = json.dumps({"context_precision_score": result})
      return result
  
  
  @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def get_context_recall(row_str: str) -> str:
      row = json.loads(row_str)
      result = metrics.context_recall.score_single(row)
      result = json.dumps({"context_recall_score": result})
      return result
  
  
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def evaluate_row(idx: Hashable, row_str: str) -> str:
      eval_result = {"idx": idx}
      row = json.loads(row_str)
      eval_result.update(row)
      eval_result.update(json.loads(get_answer_correctness(row_str)))
      eval_result.update(json.loads(get_answer_relevancy(row_str)))
      eval_result.update(json.loads(get_answer_faithfulness(row_str)))
      eval_result.update(json.loads(get_answer_similarity(row_str)))
      eval_result.update(json.loads(get_context_precision(row_str)))
      eval_result.update(json.loads(get_context_recall(row_str)))
      eval_result = json.dumps(eval_result)
      return eval_result
  
  
  @cachew(cache_path=EVAL_CACHE, logger=logger)
  def process_row(
      idx: Hashable, row_str: str, application: str = "api-eval-bharat"
  ) -> str:
      eval_record = get_eval_record(row_str, application=application)
      eval_row = evaluate_row(idx, eval_record)
      return eval_row
  
  
  def main():
      eval_results = []
  
      df = pd.read_json(
          "data/eval/wandbot_cleaned_annotated_dataset_11-12-2023.jsonl",
          lines=True,
          orient="records",
      )
      correct_df = df[
          (df["is_wandb_query"] == "YES") & (df["correctness"] == "correct")
      ]
  
      with open(
          "data/eval/wandbot-gpt-4-1106-preview-eval-v1-1.jsonl", "w+"
      ) as outfile:
          for idx, row in tqdm(correct_df.iterrows(), total=len(correct_df)):
              try:
                  row_str = row.to_json()
                  eval_row = process_row(
                      idx,
                      row_str,
                      application="wandbot-gpt-4-1106-preview-eval-v1.1-bharat",
                  )
                  outfile.write(eval_row + "\n")
                  eval_results.append(eval_row)
              except Exception as e:
                  print(e)
                  print(idx)
  
  
  if __name__ == "__main__":
      main()
  
apis: llama_index.llms.OpenAI
extract_api: [((689, 709), 'wandbot.utils.get_logger', 'get_logger', (['__name__'], {}), '(__name__)\n', (699, 709), False, 'from wandbot.utils import cachew, get_logger\n'), ((711, 731), 'nest_asyncio.apply', 'nest_asyncio.apply', ([], {}), '()\n', (729, 731), False, 'import nest_asyncio\n'), ((885, 989), 'wandbot.evaluation.eval.correctness.WandbCorrectnessEvaluator', 'WandbCorrectnessEvaluator', ([], {'service_context': 'service_context', 'eval_template': 'CORRECTNESS_EVAL_TEMPLATE'}), '(service_context=service_context, eval_template=\n    CORRECTNESS_EVAL_TEMPLATE)\n', (910, 989), False, 'from wandbot.evaluation.eval.correctness import CORRECTNESS_EVAL_TEMPLATE, WandbCorrectnessEvaluator\n'), ((1021, 1125), 'wandbot.evaluation.eval.factfulness.WandbFactfulnessEvaluator', 'WandbFactfulnessEvaluator', ([], {'service_context': 'service_context', 'eval_template': 'FACTFULNESS_EVAL_TEMPLATE'}), '(service_context=service_context, eval_template=\n    FACTFULNESS_EVAL_TEMPLATE)\n', (1046, 1125), False, 'from wandbot.evaluation.eval.factfulness import FACTFULNESS_EVAL_TEMPLATE, WandbFactfulnessEvaluator\n'), ((1154, 1254), 'wandbot.evaluation.eval.relevancy.WandbRelevancyEvaluator', 'WandbRelevancyEvaluator', ([], {'service_context': 'service_context', 'eval_template': 'RELEVANCY_EVAL_TEMPLATE'}), '(service_context=service_context, eval_template=\n    RELEVANCY_EVAL_TEMPLATE)\n', (1177, 1254), False, 'from wandbot.evaluation.eval.relevancy import RELEVANCY_EVAL_TEMPLATE, WandbRelevancyEvaluator\n'), ((1344, 1388), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (1350, 1388), False, 'from wandbot.utils import cachew, get_logger\n'), ((1743, 1787), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (1749, 1787), False, 'from wandbot.utils import cachew, get_logger\n'), ((2656, 2700), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (2662, 2700), False, 'from wandbot.utils import cachew, get_logger\n'), ((3325, 3369), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (3331, 3369), False, 'from wandbot.utils import cachew, get_logger\n'), ((3936, 3980), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (3942, 3980), False, 'from wandbot.utils import cachew, get_logger\n'), ((4556, 4600), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (4562, 4600), False, 'from wandbot.utils import cachew, get_logger\n'), ((4906, 4950), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (4912, 4950), False, 'from wandbot.utils import cachew, get_logger\n'), ((5248, 5292), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (5254, 5292), False, 'from wandbot.utils import cachew, get_logger\n'), ((5501, 5545), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (5507, 5545), False, 'from wandbot.utils import cachew, get_logger\n'), ((6158, 6202), 'wandbot.utils.cachew', 'cachew', ([], {'cache_path': 'EVAL_CACHE', 'logger': 'logger'}), '(cache_path=EVAL_CACHE, logger=logger)\n', (6164, 6202), False, 'from wandbot.utils import cachew, get_logger\n'), ((1719, 1739), 'json.dumps', 'json.dumps', (['response'], {}), '(response)\n', (1729, 1739), False, 'import json\n'), ((1878, 1897), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (1888, 1897), False, 'import json\n'), ((1981, 2001), 'json.loads', 'json.loads', (['response'], {}), '(response)\n', (1991, 2001), False, 'import json\n'), ((2274, 2294), 'json.dumps', 'json.dumps', (['response'], {}), '(response)\n', (2284, 2294), False, 'import json\n'), ((2760, 2779), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (2770, 2779), False, 'import json\n'), ((3147, 3191), 'ragas.metrics.answer_correctness.score_single', 'metrics.answer_correctness.score_single', (['row'], {}), '(row)\n', (3186, 3191), False, 'from ragas import metrics\n'), ((3205, 3223), 'json.dumps', 'json.dumps', (['result'], {}), '(result)\n', (3215, 3223), False, 'import json\n'), ((3427, 3446), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (3437, 3446), False, 'import json\n'), ((3760, 3802), 'ragas.metrics.answer_relevancy.score_single', 'metrics.answer_relevancy.score_single', (['row'], {}), '(row)\n', (3797, 3802), False, 'from ragas import metrics\n'), ((3816, 3834), 'json.dumps', 'json.dumps', (['result'], {}), '(result)\n', (3826, 3834), False, 'import json\n'), ((4041, 4060), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (4051, 4060), False, 'import json\n'), ((4384, 4422), 'ragas.metrics.faithfulness.score_single', 'metrics.faithfulness.score_single', (['row'], {}), '(row)\n', (4417, 4422), False, 'from ragas import metrics\n'), ((4436, 4454), 'json.dumps', 'json.dumps', (['result'], {}), '(result)\n', (4446, 4454), False, 'import json\n'), ((4659, 4678), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (4669, 4678), False, 'import json\n'), ((4692, 4735), 'ragas.metrics.answer_similarity.score_single', 'metrics.answer_similarity.score_single', (['row'], {}), '(row)\n', (4730, 4735), False, 'from ragas import metrics\n'), ((4749, 4804), 'json.dumps', 'json.dumps', (["{'answer_similarity_score_(ragas)': result}"], {}), "({'answer_similarity_score_(ragas)': result})\n", (4759, 4804), False, 'import json\n'), ((5009, 5028), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (5019, 5028), False, 'import json\n'), ((5042, 5085), 'ragas.metrics.context_precision.score_single', 'metrics.context_precision.score_single', (['row'], {}), '(row)\n', (5080, 5085), False, 'from ragas import metrics\n'), ((5099, 5146), 'json.dumps', 'json.dumps', (["{'context_precision_score': result}"], {}), "({'context_precision_score': result})\n", (5109, 5146), False, 'import json\n'), ((5348, 5367), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (5358, 5367), False, 'import json\n'), ((5381, 5421), 'ragas.metrics.context_recall.score_single', 'metrics.context_recall.score_single', (['row'], {}), '(row)\n', (5416, 5421), False, 'from ragas import metrics\n'), ((5435, 5479), 'json.dumps', 'json.dumps', (["{'context_recall_score': result}"], {}), "({'context_recall_score': result})\n", (5445, 5479), False, 'import json\n'), ((5641, 5660), 'json.loads', 'json.loads', (['row_str'], {}), '(row_str)\n', (5651, 5660), False, 'import json\n'), ((6108, 6131), 'json.dumps', 'json.dumps', (['eval_result'], {}), '(eval_result)\n', (6118, 6131), False, 'import json\n'), ((6480, 6590), 'pandas.read_json', 'pd.read_json', (['"""data/eval/wandbot_cleaned_annotated_dataset_11-12-2023.jsonl"""'], {'lines': '(True)', 'orient': '"""records"""'}), "('data/eval/wandbot_cleaned_annotated_dataset_11-12-2023.jsonl',\n    lines=True, orient='records')\n", (6492, 6590), True, 'import pandas as pd\n'), ((831, 859), 'llama_index.llms.OpenAI', 'OpenAI', (['"""gpt-4-1106-preview"""'], {}), "('gpt-4-1106-preview')\n", (837, 859), False, 'from llama_index.llms import OpenAI\n'), ((1275, 1313), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (1298, 1313), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((1320, 1341), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (1338, 1341), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((2587, 2625), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (2610, 2625), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((2632, 2653), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (2650, 2653), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((3256, 3294), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (3279, 3294), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((3301, 3322), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (3319, 3322), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((3867, 3905), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (3890, 3905), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((3912, 3933), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (3930, 3933), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((4487, 4525), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (4510, 4525), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((4532, 4553), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (4550, 4553), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((4837, 4875), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (4860, 4875), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((4882, 4903), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (4900, 4903), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((5179, 5217), 'tenacity.wait_random_exponential', 'wait_random_exponential', ([], {'min': '(1)', 'max': '(60)'}), '(min=1, max=60)\n', (5202, 5217), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((5224, 5245), 'tenacity.stop_after_attempt', 'stop_after_attempt', (['(6)'], {}), '(6)\n', (5242, 5245), False, 'from tenacity import retry, stop_after_attempt, wait_random_exponential\n'), ((1656, 1675), 'json.dumps', 'json.dumps', (['payload'], {}), '(payload)\n', (1666, 1675), False, 'import json\n'), ((2212, 2252), 'json.loads', 'json.loads', (["response['source_documents']"], {}), "(response['source_documents'])\n", (2222, 2252), False, 'import json\n')]