code: import os
  from dotenv import load_dotenv, find_dotenv
  
  import numpy as np
  
  import nest_asyncio
  
  nest_asyncio.apply()
  
  
  def get_openai_api_key():
      _ = load_dotenv(find_dotenv())
  
      return os.getenv("OPENAI_API_KEY")
  
  from trulens_eval import (
      Feedback,
      TruLlama,
      OpenAI
  )
  
  from trulens_eval.feedback import Groundedness
  
  def get_prebuilt_trulens_recorder(query_engine, app_id):
      openai = OpenAI()
  
      qa_relevance = (
          Feedback(openai.relevance_with_cot_reasons, name="Answer Relevance")
          .on_input_output()
      )
  
      qs_relevance = (
          Feedback(openai.relevance_with_cot_reasons, name = "Context Relevance")
          .on_input()
          .on(TruLlama.select_source_nodes().node.text)
          .aggregate(np.mean)
      )
  
  #     grounded = Groundedness(groundedness_provider=openai, summarize_provider=openai)
      grounded = Groundedness(groundedness_provider=openai)
  
      groundedness = (
          Feedback(grounded.groundedness_measure_with_cot_reasons, name="Groundedness")
              .on(TruLlama.select_source_nodes().node.text)
              .on_output()
              .aggregate(grounded.grounded_statements_aggregator)
      )
  
      feedbacks = [qa_relevance, qs_relevance, groundedness]
      tru_recorder = TruLlama(
          query_engine,
          app_id=app_id,
          feedbacks=feedbacks
      )
      return tru_recorder
  
  from llama_index import ServiceContext, VectorStoreIndex, StorageContext
  from llama_index.node_parser import SentenceWindowNodeParser
  from llama_index.indices.postprocessor import MetadataReplacementPostProcessor
  from llama_index.indices.postprocessor import SentenceTransformerRerank
  from llama_index import load_index_from_storage
  import os
  
  
  def build_sentence_window_index(
      documents,
      llm,
      embed_model="local:BAAI/bge-small-en-v1.5",
      sentence_window_size=3,
      save_dir="sentence_index",
  ):
      # create the sentence window node parser w/ default settings
      node_parser = SentenceWindowNodeParser.from_defaults(
          window_size=sentence_window_size,
          window_metadata_key="window",
          original_text_metadata_key="original_text",
      )
      sentence_context = ServiceContext.from_defaults(
          llm=llm,
          embed_model=embed_model,
          node_parser=node_parser,
      )
      if not os.path.exists(save_dir):
          sentence_index = VectorStoreIndex.from_documents(
              documents, service_context=sentence_context
          )
          sentence_index.storage_context.persist(persist_dir=save_dir)
      else:
          sentence_index = load_index_from_storage(
              StorageContext.from_defaults(persist_dir=save_dir),
              service_context=sentence_context,
          )
  
      return sentence_index
  
  
  def get_sentence_window_query_engine(
      sentence_index,
      similarity_top_k=6,
      rerank_top_n=2,
  ):
      # define postprocessors
      postproc = MetadataReplacementPostProcessor(target_metadata_key="window")
      rerank = SentenceTransformerRerank(
          top_n=rerank_top_n, model="BAAI/bge-reranker-base"
      )
  
      sentence_window_engine = sentence_index.as_query_engine(
          similarity_top_k=similarity_top_k, node_postprocessors=[postproc, rerank]
      )
      return sentence_window_engine
  
  
  from llama_index.node_parser import HierarchicalNodeParser
  
  from llama_index.node_parser import get_leaf_nodes
  from llama_index import StorageContext
  from llama_index.retrievers import AutoMergingRetriever
  from llama_index.indices.postprocessor import SentenceTransformerRerank
  from llama_index.query_engine import RetrieverQueryEngine
  
  
  def build_automerging_index(
      documents,
      llm,
      embed_model="local:BAAI/bge-small-en-v1.5",
      save_dir="merging_index",
      chunk_sizes=None,
  ):
      chunk_sizes = chunk_sizes or [2048, 512, 128]
      node_parser = HierarchicalNodeParser.from_defaults(chunk_sizes=chunk_sizes)
      nodes = node_parser.get_nodes_from_documents(documents)
      leaf_nodes = get_leaf_nodes(nodes)
      merging_context = ServiceContext.from_defaults(
          llm=llm,
          embed_model=embed_model,
      )
      storage_context = StorageContext.from_defaults()
      storage_context.docstore.add_documents(nodes)
  
      if not os.path.exists(save_dir):
          automerging_index = VectorStoreIndex(
              leaf_nodes, storage_context=storage_context, service_context=merging_context
          )
          automerging_index.storage_context.persist(persist_dir=save_dir)
      else:
          automerging_index = load_index_from_storage(
              StorageContext.from_defaults(persist_dir=save_dir),
              service_context=merging_context,
          )
      return automerging_index
  
  
  def get_automerging_query_engine(
      automerging_index,
      similarity_top_k=12,
      rerank_top_n=6,
  ):
      base_retriever = automerging_index.as_retriever(similarity_top_k=similarity_top_k)
      retriever = AutoMergingRetriever(
          base_retriever, automerging_index.storage_context, verbose=True
      )
      rerank = SentenceTransformerRerank(
          top_n=rerank_top_n, model="BAAI/bge-reranker-base"
      )
      auto_merging_engine = RetrieverQueryEngine.from_args(
          retriever, node_postprocessors=[rerank]
      )
      return auto_merging_engine
  
apis: llama_index.VectorStoreIndex.from_documents llama_index.retrievers.AutoMergingRetriever llama_index.node_parser.get_leaf_nodes llama_index.ServiceContext.from_defaults llama_index.StorageContext.from_defaults llama_index.node_parser.SentenceWindowNodeParser.from_defaults llama_index.VectorStoreIndex llama_index.node_parser.HierarchicalNodeParser.from_defaults llama_index.query_engine.RetrieverQueryEngine.from_args llama_index.indices.postprocessor.MetadataReplacementPostProcessor llama_index.indices.postprocessor.SentenceTransformerRerank
extract_api: [((96, 116), 'nest_asyncio.apply', 'nest_asyncio.apply', ([], {}), '()\n', (114, 116), False, 'import nest_asyncio\n'), ((192, 219), 'os.getenv', 'os.getenv', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (201, 219), False, 'import os\n'), ((408, 416), 'trulens_eval.OpenAI', 'OpenAI', ([], {}), '()\n', (414, 416), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n'), ((862, 904), 'trulens_eval.feedback.Groundedness', 'Groundedness', ([], {'groundedness_provider': 'openai'}), '(groundedness_provider=openai)\n', (874, 904), False, 'from trulens_eval.feedback import Groundedness\n'), ((1245, 1303), 'trulens_eval.TruLlama', 'TruLlama', (['query_engine'], {'app_id': 'app_id', 'feedbacks': 'feedbacks'}), '(query_engine, app_id=app_id, feedbacks=feedbacks)\n', (1253, 1303), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n'), ((1954, 2104), 'llama_index.node_parser.SentenceWindowNodeParser.from_defaults', 'SentenceWindowNodeParser.from_defaults', ([], {'window_size': 'sentence_window_size', 'window_metadata_key': '"""window"""', 'original_text_metadata_key': '"""original_text"""'}), "(window_size=sentence_window_size,\n    window_metadata_key='window', original_text_metadata_key='original_text')\n", (1992, 2104), False, 'from llama_index.node_parser import SentenceWindowNodeParser\n'), ((2155, 2247), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'llm': 'llm', 'embed_model': 'embed_model', 'node_parser': 'node_parser'}), '(llm=llm, embed_model=embed_model, node_parser=\n    node_parser)\n', (2183, 2247), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext\n'), ((2861, 2923), 'llama_index.indices.postprocessor.MetadataReplacementPostProcessor', 'MetadataReplacementPostProcessor', ([], {'target_metadata_key': '"""window"""'}), "(target_metadata_key='window')\n", (2893, 2923), False, 'from llama_index.indices.postprocessor import MetadataReplacementPostProcessor\n'), ((2937, 3014), 'llama_index.indices.postprocessor.SentenceTransformerRerank', 'SentenceTransformerRerank', ([], {'top_n': 'rerank_top_n', 'model': '"""BAAI/bge-reranker-base"""'}), "(top_n=rerank_top_n, model='BAAI/bge-reranker-base')\n", (2962, 3014), False, 'from llama_index.indices.postprocessor import SentenceTransformerRerank\n'), ((3777, 3838), 'llama_index.node_parser.HierarchicalNodeParser.from_defaults', 'HierarchicalNodeParser.from_defaults', ([], {'chunk_sizes': 'chunk_sizes'}), '(chunk_sizes=chunk_sizes)\n', (3813, 3838), False, 'from llama_index.node_parser import HierarchicalNodeParser\n'), ((3916, 3937), 'llama_index.node_parser.get_leaf_nodes', 'get_leaf_nodes', (['nodes'], {}), '(nodes)\n', (3930, 3937), False, 'from llama_index.node_parser import get_leaf_nodes\n'), ((3960, 4022), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'llm': 'llm', 'embed_model': 'embed_model'}), '(llm=llm, embed_model=embed_model)\n', (3988, 4022), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext\n'), ((4068, 4098), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {}), '()\n', (4096, 4098), False, 'from llama_index import StorageContext\n'), ((4825, 4914), 'llama_index.retrievers.AutoMergingRetriever', 'AutoMergingRetriever', (['base_retriever', 'automerging_index.storage_context'], {'verbose': '(True)'}), '(base_retriever, automerging_index.storage_context,\n    verbose=True)\n', (4845, 4914), False, 'from llama_index.retrievers import AutoMergingRetriever\n'), ((4938, 5015), 'llama_index.indices.postprocessor.SentenceTransformerRerank', 'SentenceTransformerRerank', ([], {'top_n': 'rerank_top_n', 'model': '"""BAAI/bge-reranker-base"""'}), "(top_n=rerank_top_n, model='BAAI/bge-reranker-base')\n", (4963, 5015), False, 'from llama_index.indices.postprocessor import SentenceTransformerRerank\n'), ((5056, 5127), 'llama_index.query_engine.RetrieverQueryEngine.from_args', 'RetrieverQueryEngine.from_args', (['retriever'], {'node_postprocessors': '[rerank]'}), '(retriever, node_postprocessors=[rerank])\n', (5086, 5127), False, 'from llama_index.query_engine import RetrieverQueryEngine\n'), ((165, 178), 'dotenv.find_dotenv', 'find_dotenv', ([], {}), '()\n', (176, 178), False, 'from dotenv import load_dotenv, find_dotenv\n'), ((2285, 2309), 'os.path.exists', 'os.path.exists', (['save_dir'], {}), '(save_dir)\n', (2299, 2309), False, 'import os\n'), ((2336, 2412), 'llama_index.VectorStoreIndex.from_documents', 'VectorStoreIndex.from_documents', (['documents'], {'service_context': 'sentence_context'}), '(documents, service_context=sentence_context)\n', (2367, 2412), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext\n'), ((4161, 4185), 'os.path.exists', 'os.path.exists', (['save_dir'], {}), '(save_dir)\n', (4175, 4185), False, 'import os\n'), ((4215, 4313), 'llama_index.VectorStoreIndex', 'VectorStoreIndex', (['leaf_nodes'], {'storage_context': 'storage_context', 'service_context': 'merging_context'}), '(leaf_nodes, storage_context=storage_context,\n    service_context=merging_context)\n', (4231, 4313), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext\n'), ((447, 515), 'trulens_eval.Feedback', 'Feedback', (['openai.relevance_with_cot_reasons'], {'name': '"""Answer Relevance"""'}), "(openai.relevance_with_cot_reasons, name='Answer Relevance')\n", (455, 515), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n'), ((2576, 2626), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': 'save_dir'}), '(persist_dir=save_dir)\n', (2604, 2626), False, 'from llama_index import StorageContext\n'), ((4479, 4529), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': 'save_dir'}), '(persist_dir=save_dir)\n', (4507, 4529), False, 'from llama_index import StorageContext\n'), ((683, 713), 'trulens_eval.TruLlama.select_source_nodes', 'TruLlama.select_source_nodes', ([], {}), '()\n', (711, 713), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n'), ((579, 648), 'trulens_eval.Feedback', 'Feedback', (['openai.relevance_with_cot_reasons'], {'name': '"""Context Relevance"""'}), "(openai.relevance_with_cot_reasons, name='Context Relevance')\n", (587, 648), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n'), ((935, 1012), 'trulens_eval.Feedback', 'Feedback', (['grounded.groundedness_measure_with_cot_reasons'], {'name': '"""Groundedness"""'}), "(grounded.groundedness_measure_with_cot_reasons, name='Groundedness')\n", (943, 1012), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n'), ((1029, 1059), 'trulens_eval.TruLlama.select_source_nodes', 'TruLlama.select_source_nodes', ([], {}), '()\n', (1057, 1059), False, 'from trulens_eval import Feedback, TruLlama, OpenAI\n')]