code: import os
  from shutil import rmtree
  from typing import Callable, Dict, List, Optional
  
  import tqdm
  from llama_index.core.base.base_retriever import BaseRetriever
  from llama_index.core.postprocessor.types import BaseNodePostprocessor
  from llama_index.core.schema import Document, QueryBundle
  from llama_index.core.utils import get_cache_dir
  
  
  class BeirEvaluator:
      """
      Refer to: https://github.com/beir-cellar/beir for a full list of supported datasets
      and a full description of BEIR.
      """
  
      def __init__(self) -> None:
          try:
              pass
          except ImportError:
              raise ImportError(
                  "Please install beir to use this feature: " "`pip install beir`",
              )
  
      def _download_datasets(self, datasets: List[str] = ["nfcorpus"]) -> Dict[str, str]:
          from beir import util
  
          cache_dir = get_cache_dir()
  
          dataset_paths = {}
          for dataset in datasets:
              dataset_full_path = os.path.join(cache_dir, "datasets", "BeIR__" + dataset)
              if not os.path.exists(dataset_full_path):
                  url = f"""https://public.ukp.informatik.tu-darmstadt.de/thakur\
  /BEIR/datasets/{dataset}.zip"""
                  try:
                      util.download_and_unzip(url, dataset_full_path)
                  except Exception as e:
                      print(
                          "Dataset:", dataset, "not found at:", url, "Removing cached dir"
                      )
                      rmtree(dataset_full_path)
                      raise ValueError(f"invalid BEIR dataset: {dataset}") from e
  
              print("Dataset:", dataset, "downloaded at:", dataset_full_path)
              dataset_paths[dataset] = os.path.join(dataset_full_path, dataset)
          return dataset_paths
  
      def run(
          self,
          create_retriever: Callable[[List[Document]], BaseRetriever],
          datasets: List[str] = ["nfcorpus"],
          metrics_k_values: List[int] = [3, 10],
          node_postprocessors: Optional[List[BaseNodePostprocessor]] = None,
      ) -> None:
          from beir.datasets.data_loader import GenericDataLoader
          from beir.retrieval.evaluation import EvaluateRetrieval
  
          dataset_paths = self._download_datasets(datasets)
          for dataset in datasets:
              dataset_path = dataset_paths[dataset]
              print("Evaluating on dataset:", dataset)
              print("-------------------------------------")
  
              corpus, queries, qrels = GenericDataLoader(data_folder=dataset_path).load(
                  split="test"
              )
  
              documents = []
              for id, val in corpus.items():
                  doc = Document(
                      text=val["text"], metadata={"title": val["title"], "doc_id": id}
                  )
                  documents.append(doc)
  
              retriever = create_retriever(documents)
  
              print("Retriever created for: ", dataset)
  
              print("Evaluating retriever on questions against qrels")
  
              results = {}
              for key, query in tqdm.tqdm(queries.items()):
                  nodes_with_score = retriever.retrieve(query)
                  node_postprocessors = node_postprocessors or []
                  for node_postprocessor in node_postprocessors:
                      nodes_with_score = node_postprocessor.postprocess_nodes(
                          nodes_with_score, query_bundle=QueryBundle(query_str=query)
                      )
                  results[key] = {
                      node.node.metadata["doc_id"]: node.score
                      for node in nodes_with_score
                  }
  
              ndcg, map_, recall, precision = EvaluateRetrieval.evaluate(
                  qrels, results, metrics_k_values
              )
              print("Results for:", dataset)
              for k in metrics_k_values:
                  print(
                      {
                          f"NDCG@{k}": ndcg[f"NDCG@{k}"],
                          f"MAP@{k}": map_[f"MAP@{k}"],
                          f"Recall@{k}": recall[f"Recall@{k}"],
                          f"precision@{k}": precision[f"P@{k}"],
                      }
                  )
              print("-------------------------------------")
  
apis: llama_index.core.utils.get_cache_dir llama_index.core.schema.QueryBundle llama_index.core.schema.Document
extract_api: [((861, 876), 'llama_index.core.utils.get_cache_dir', 'get_cache_dir', ([], {}), '()\n', (874, 876), False, 'from llama_index.core.utils import get_cache_dir\n'), ((970, 1025), 'os.path.join', 'os.path.join', (['cache_dir', '"""datasets"""', "('BeIR__' + dataset)"], {}), "(cache_dir, 'datasets', 'BeIR__' + dataset)\n", (982, 1025), False, 'import os\n'), ((1698, 1738), 'os.path.join', 'os.path.join', (['dataset_full_path', 'dataset'], {}), '(dataset_full_path, dataset)\n', (1710, 1738), False, 'import os\n'), ((3642, 3702), 'beir.retrieval.evaluation.EvaluateRetrieval.evaluate', 'EvaluateRetrieval.evaluate', (['qrels', 'results', 'metrics_k_values'], {}), '(qrels, results, metrics_k_values)\n', (3668, 3702), False, 'from beir.retrieval.evaluation import EvaluateRetrieval\n'), ((1045, 1078), 'os.path.exists', 'os.path.exists', (['dataset_full_path'], {}), '(dataset_full_path)\n', (1059, 1078), False, 'import os\n'), ((2652, 2726), 'llama_index.core.schema.Document', 'Document', ([], {'text': "val['text']", 'metadata': "{'title': val['title'], 'doc_id': id}"}), "(text=val['text'], metadata={'title': val['title'], 'doc_id': id})\n", (2660, 2726), False, 'from llama_index.core.schema import Document, QueryBundle\n'), ((1233, 1280), 'beir.util.download_and_unzip', 'util.download_and_unzip', (['url', 'dataset_full_path'], {}), '(url, dataset_full_path)\n', (1256, 1280), False, 'from beir import util\n'), ((2466, 2509), 'beir.datasets.data_loader.GenericDataLoader', 'GenericDataLoader', ([], {'data_folder': 'dataset_path'}), '(data_folder=dataset_path)\n', (2483, 2509), False, 'from beir.datasets.data_loader import GenericDataLoader\n'), ((1478, 1503), 'shutil.rmtree', 'rmtree', (['dataset_full_path'], {}), '(dataset_full_path)\n', (1484, 1503), False, 'from shutil import rmtree\n'), ((3385, 3413), 'llama_index.core.schema.QueryBundle', 'QueryBundle', ([], {'query_str': 'query'}), '(query_str=query)\n', (3396, 3413), False, 'from llama_index.core.schema import Document, QueryBundle\n')]