code: import asyncio
  
  from llama_index.core.llama_dataset import download_llama_dataset
  from llama_index.core.llama_pack import download_llama_pack
  from llama_index.core.evaluation import CorrectnessEvaluator
  from llama_index.llms import OpenAI, Gemini
  from llama_index.core import ServiceContext
  import pandas as pd
  
  
  async def main():
      # DOWNLOAD LLAMADATASET
      evaluator_dataset, _ = download_llama_dataset(
          "MiniMtBenchSingleGradingDataset", "./mini_mt_bench_data"
      )
  
      # DEFINE EVALUATORS
      gpt_4_context = ServiceContext.from_defaults(
          llm=OpenAI(temperature=0, model="gpt-4"),
      )
  
      gpt_3p5_context = ServiceContext.from_defaults(
          llm=OpenAI(temperature=0, model="gpt-3.5-turbo"),
      )
  
      gemini_pro_context = ServiceContext.from_defaults(
          llm=Gemini(model="models/gemini-pro", temperature=0)
      )
  
      evaluators = {
          "gpt-4": CorrectnessEvaluator(service_context=gpt_4_context),
          "gpt-3.5": CorrectnessEvaluator(service_context=gpt_3p5_context),
          "gemini-pro": CorrectnessEvaluator(service_context=gemini_pro_context),
      }
  
      # EVALUATE WITH PACK
      ############################################################################
      # NOTE: If have a lower tier subscription for OpenAI API like Usage Tier 1 #
      # then you'll need to use different batch_size and sleep_time_in_seconds.  #
      # For Usage Tier 1, settings that seemed to work well were batch_size=5,   #
      # and sleep_time_in_seconds=15 (as of December 2023.)                      #
      ############################################################################
      EvaluatorBenchmarkerPack = download_llama_pack("EvaluatorBenchmarkerPack", "./pack")
      evaluator_benchmarker = EvaluatorBenchmarkerPack(
          evaluator=evaluators["gpt-3.5"],
          eval_dataset=evaluator_dataset,
          show_progress=True,
      )
      gpt_3p5_benchmark_df = await evaluator_benchmarker.arun(
          batch_size=100, sleep_time_in_seconds=0
      )
  
      evaluator_benchmarker = EvaluatorBenchmarkerPack(
          evaluator=evaluators["gpt-4"],
          eval_dataset=evaluator_dataset,
          show_progress=True,
      )
      gpt_4_benchmark_df = await evaluator_benchmarker.arun(
          batch_size=100, sleep_time_in_seconds=0
      )
  
      evaluator_benchmarker = EvaluatorBenchmarkerPack(
          evaluator=evaluators["gemini-pro"],
          eval_dataset=evaluator_dataset,
          show_progress=True,
      )
      gemini_pro_benchmark_df = await evaluator_benchmarker.arun(
          batch_size=5, sleep_time_in_seconds=0.5
      )
  
      benchmark_df = pd.concat(
          [
              gpt_3p5_benchmark_df,
              gpt_4_benchmark_df,
              gemini_pro_benchmark_df,
          ],
          axis=0,
      )
      print(benchmark_df)
  
  
  if __name__ == "__main__":
      loop = asyncio.get_event_loop()
      loop.run_until_complete(main)
  
apis: llama_index.llms.Gemini llama_index.core.evaluation.CorrectnessEvaluator llama_index.core.llama_dataset.download_llama_dataset llama_index.core.llama_pack.download_llama_pack llama_index.llms.OpenAI
extract_api: [((386, 471), 'llama_index.core.llama_dataset.download_llama_dataset', 'download_llama_dataset', (['"""MiniMtBenchSingleGradingDataset"""', '"""./mini_mt_bench_data"""'], {}), "('MiniMtBenchSingleGradingDataset',\n    './mini_mt_bench_data')\n", (408, 471), False, 'from llama_index.core.llama_dataset import download_llama_dataset\n'), ((1646, 1703), 'llama_index.core.llama_pack.download_llama_pack', 'download_llama_pack', (['"""EvaluatorBenchmarkerPack"""', '"""./pack"""'], {}), "('EvaluatorBenchmarkerPack', './pack')\n", (1665, 1703), False, 'from llama_index.core.llama_pack import download_llama_pack\n'), ((2580, 2670), 'pandas.concat', 'pd.concat', (['[gpt_3p5_benchmark_df, gpt_4_benchmark_df, gemini_pro_benchmark_df]'], {'axis': '(0)'}), '([gpt_3p5_benchmark_df, gpt_4_benchmark_df,\n    gemini_pro_benchmark_df], axis=0)\n', (2589, 2670), True, 'import pandas as pd\n'), ((2801, 2825), 'asyncio.get_event_loop', 'asyncio.get_event_loop', ([], {}), '()\n', (2823, 2825), False, 'import asyncio\n'), ((890, 941), 'llama_index.core.evaluation.CorrectnessEvaluator', 'CorrectnessEvaluator', ([], {'service_context': 'gpt_4_context'}), '(service_context=gpt_4_context)\n', (910, 941), False, 'from llama_index.core.evaluation import CorrectnessEvaluator\n'), ((962, 1015), 'llama_index.core.evaluation.CorrectnessEvaluator', 'CorrectnessEvaluator', ([], {'service_context': 'gpt_3p5_context'}), '(service_context=gpt_3p5_context)\n', (982, 1015), False, 'from llama_index.core.evaluation import CorrectnessEvaluator\n'), ((1039, 1095), 'llama_index.core.evaluation.CorrectnessEvaluator', 'CorrectnessEvaluator', ([], {'service_context': 'gemini_pro_context'}), '(service_context=gemini_pro_context)\n', (1059, 1095), False, 'from llama_index.core.evaluation import CorrectnessEvaluator\n'), ((569, 605), 'llama_index.llms.OpenAI', 'OpenAI', ([], {'temperature': '(0)', 'model': '"""gpt-4"""'}), "(temperature=0, model='gpt-4')\n", (575, 605), False, 'from llama_index.llms import OpenAI, Gemini\n'), ((678, 722), 'llama_index.llms.OpenAI', 'OpenAI', ([], {'temperature': '(0)', 'model': '"""gpt-3.5-turbo"""'}), "(temperature=0, model='gpt-3.5-turbo')\n", (684, 722), False, 'from llama_index.llms import OpenAI, Gemini\n'), ((798, 846), 'llama_index.llms.Gemini', 'Gemini', ([], {'model': '"""models/gemini-pro"""', 'temperature': '(0)'}), "(model='models/gemini-pro', temperature=0)\n", (804, 846), False, 'from llama_index.llms import OpenAI, Gemini\n')]