code: from importlib import metadata
  from pathlib import WindowsPath
  from re import sub
  from llama_index import (
      ServiceContext,
      VectorStoreIndex,
      StorageContext,
      load_index_from_storage,
      global_service_context,
  )
  import llama_index
  from llama_index.embeddings import HuggingFaceEmbedding
  from llama_index.schema import TextNode, MetadataMode
  from llama_index.vector_stores import MilvusVectorStore
  from llama_index.readers import SimpleDirectoryReader
  from llama_index.node_parser import SentenceWindowNodeParser, SentenceSplitter, SimpleNodeParser
  from llama_index.postprocessor import (
      SimilarityPostprocessor,
      MetadataReplacementPostProcessor,
  )
  
  from milvus import default_server
  
  import os
  from typing import List, Dict, Any, Optional
  
  
  from chatbot.common import DATA_PATH, EMBEDDING_DIM, EMBEDDING_MODEL, SIMILARITY_SEARCH_THRESHOLD, path_leaf, subjects, PathSep, debug
  
  
  class AugmentedIngestPipeline:
      def __init__(
          self, data_dir_path: str, service_context: ServiceContext,
      ) -> None:
          self.data_dir = data_dir_path
          self.service_ctx = service_context
          self.embed_model = self.service_ctx.embed_model
          self.vector_indexes = {}
          self.metadata_fn = lambda x: {"title": x.replace("_", " ")}
          self.node_parser = SentenceWindowNodeParser.from_defaults(
              window_size=3,
              window_metadata_key="window",
              original_text_metadata_key="original_text",
              include_metadata=True,
          )
          self.create = False
  
      def _load_data(self, path):
          docs = SimpleDirectoryReader(
              path, file_metadata=self.metadata_fn, filename_as_id=True
          ).load_data()
          return docs
  
      def _make_nodes(self, docs):
          nodes = self.node_parser.get_nodes_from_documents(docs, show_progress=debug)
          return nodes
  
      def _insert_into_vectorstore(self, subject, nodes, create=False):
          collection_name = f"augmentED_{subject}"
          vector_store = MilvusVectorStore(
              dim=EMBEDDING_DIM,
              host="127.0.0.1",
              port=default_server.listen_port,
              collection_name=collection_name,
              overwrite=create,
          )
  
          storage_ctx = StorageContext.from_defaults(vector_store=vector_store)
  
          self.vector_indexes[subject] = VectorStoreIndex(
              nodes=nodes,
              service_context=self.service_ctx,
              storage_context=storage_ctx,
          )
  
      def _load_vectorstore(self, subject):
          collection_name = f"augmentED_{subject}"
          vector_store = MilvusVectorStore(
              dim=EMBEDDING_DIM,
              host="127.0.0.1",
              port=default_server.listen_port,
              collection_name=collection_name,
              overwrite=False
          )
  
          storage_ctx = StorageContext.from_defaults(vector_store=vector_store)
  
          self.vector_indexes[subject] = VectorStoreIndex.from_vector_store(
              vector_store=vector_store,
              service_context=self.service_ctx,
              storage_context=storage_ctx,
          )
  
      def _get_subject_query_engine(self, subject):
          query_engine = self.vector_indexes[subject].as_query_engine(
              similarity_top_k=3,
              node_postprocessors=[
                  SimilarityPostprocessor(similarity_cutoff=SIMILARITY_SEARCH_THRESHOLD),
                  MetadataReplacementPostProcessor(target_metadata_key="window")
              ],
          )
          return query_engine
  
      def _get_subject_chat_engine(self, subject):
          query_engine = self.vector_indexes[subject].as_chat_engine(
              mode="context",
              similarity_top_k=2,
              node_postprocessors=[
                  SimilarityPostprocessor(similarity_cutoff=SIMILARITY_SEARCH_THRESHOLD),
                  MetadataReplacementPostProcessor(target_metadata_key="window")
              ],
          )
          return query_engine
  
  
      def run_pipeline(self, create=False):
          self.create = create
          if self.create:
              self.one_giant_index_nodes = []
              self.all_docs = []
              for subject in subjects:
                  path = self.data_dir + PathSep + subjects[subject]
  
                  docs = self._load_data(path)
                  nodes = self._make_nodes(docs)
                  self._insert_into_vectorstore(subject=subject, nodes=nodes)
  
                  self.one_giant_index_nodes.extend(nodes)
                  self.all_docs.extend(docs)
  
              self._insert_into_vectorstore(
                  subject="OneGiantIndex", nodes=self.one_giant_index_nodes, create=self.create
              )
  
          else:
              for subject in subjects:
                  self._load_vectorstore(subject)
              self._load_vectorstore("OneGiantIndex")
  
          self.one_giant_index = self.vector_indexes["OneGiantIndex"]
          self.query_everything = self._get_subject_query_engine("OneGiantIndex")
      
      def search_one_giant_index(
          self,
          query,
          top_k=10,
          replace_with_meta=True,
          metadata_key="title",
      ):
          retr = self.one_giant_index.as_retriever(
              similarity_top_k=top_k,
          )
          answers = retr.retrieve(query)
          if replace_with_meta:
              return list(set(map(lambda x: x.metadata[metadata_key], answers)))
          else:
              return list(
                  map(lambda x: x.get_content(metadata_mode=MetadataMode.LLM), answers)
              )
  
      def query_one_file(self,file_path):
          docs = SimpleDirectoryReader(
              input_files=[file_path], file_metadata=self.metadata_fn, filename_as_id=True
          ).load_data()
          nodes = self._make_nodes(docs)
          self._insert_into_vectorstore("UserUploadedDocument", nodes)
          self._insert_into_vectorstore("OneGiantIndex", nodes)
          return self._get_subject_query_engine("UserUploadedDocument")
      
  
  class SimpleIngestPipeline:
      def __init__(
          self, data_dir_path: str, service_context: ServiceContext, create=False
      ) -> None:
          self.data_dir = data_dir_path
          self.service_ctx = service_context
          self.embed_model = self.service_ctx.embed_model
          self.vector_indexes = {}
          self.metadata_fn = lambda x: {"title": path_leaf(x)}
          self.node_parser = SimpleNodeParser(chunk_size=512)
          self.create = create
  
      def _load_data(self, path):
          docs = SimpleDirectoryReader(
              path, file_metadata=self.metadata_fn, filename_as_id=True
          ).load_data()
          return docs
  
      def _make_nodes(self, docs):
          nodes = self.node_parser.get_nodes_from_documents(docs, show_progress=debug)
          return nodes
  
      def _insert_into_vectorstore(self, subject, nodes, create=False):
          collection_name = f"augmentED_{subject}"
          vector_store = MilvusVectorStore(
              dim=EMBEDDING_DIM,
              host="127.0.0.1",
              port=default_server.listen_port,
              collection_name=collection_name,
              overwrite=create,
          )
  
          storage_ctx = StorageContext.from_defaults(vector_store=vector_store)
  
          self.vector_indexes[subject] = VectorStoreIndex(
              nodes=nodes,
              service_context=self.service_ctx,
              storage_context=storage_ctx,
          )
  
      def _load_vectorstore(self, subject):
          collection_name = f"augmentED_{subject}"
          vector_store = MilvusVectorStore(
              dim=EMBEDDING_DIM,
              host="127.0.0.1",
              port=default_server.listen_port,
              collection_name=collection_name,
              overwrite=False
          )
  
          storage_ctx = StorageContext.from_defaults(vector_store=vector_store)
  
          self.vector_indexes[subject] = VectorStoreIndex.from_vector_store(
              vector_store=vector_store,
              service_context=self.service_ctx,
              storage_context=storage_ctx,
          )
  
  
      def _get_subject_query_engine(self, subject) -> Dict:
          query_engine = self.vector_indexes[subject].as_query_engine(
              similarity_top_k=3,
              node_postprocessors=[
                  MetadataReplacementPostProcessor(target_metadata_key="window")
              ],
          )
          return query_engine
  
      def run_pipeline(self):
          if self.create:
              self.one_giant_index_nodes = []
              self.all_docs = []
              for subject in subjects:
                  path = self.data_dir + PathSep + subjects[subject]
  
                  docs = self._load_data(path)
                  nodes = self._make_nodes(docs)
                  self._insert_into_vectorstore(subject=subject, nodes=nodes)
  
                  self.one_giant_index_nodes.extend(nodes)
                  self.all_docs.extend(docs)
  
              self._insert_into_vectorstore(
                  subject="OneGiantIndex", nodes=self.one_giant_index_nodes, create=self.create
              )
  
          else:
              for subject in subjects:
                  self._load_vectorstore(subject)
              self._load_vectorstore("OneGiantIndex")
  
          self.one_giant_index = self.vector_indexes["OneGiantIndex"]
  
  
  if __name__ == "__main__":
      pipe = AugmentedIngestPipeline(
          data_dir_path=DATA_PATH,
          service_context=ServiceContext.from_defaults(
              llm=None, embed_model=HuggingFaceEmbedding(EMBEDDING_MODEL)
          ),
      )
      pipe.run_pipeline(create=True)
  
apis: llama_index.postprocessor.MetadataReplacementPostProcessor llama_index.postprocessor.SimilarityPostprocessor llama_index.node_parser.SimpleNodeParser llama_index.StorageContext.from_defaults llama_index.VectorStoreIndex llama_index.readers.SimpleDirectoryReader llama_index.VectorStoreIndex.from_vector_store llama_index.vector_stores.MilvusVectorStore llama_index.embeddings.HuggingFaceEmbedding llama_index.node_parser.SentenceWindowNodeParser.from_defaults
extract_api: [((1300, 1460), 'llama_index.node_parser.SentenceWindowNodeParser.from_defaults', 'SentenceWindowNodeParser.from_defaults', ([], {'window_size': '(3)', 'window_metadata_key': '"""window"""', 'original_text_metadata_key': '"""original_text"""', 'include_metadata': '(True)'}), "(window_size=3, window_metadata_key=\n    'window', original_text_metadata_key='original_text', include_metadata=True\n    )\n", (1338, 1460), False, 'from llama_index.node_parser import SentenceWindowNodeParser, SentenceSplitter, SimpleNodeParser\n'), ((2004, 2147), 'llama_index.vector_stores.MilvusVectorStore', 'MilvusVectorStore', ([], {'dim': 'EMBEDDING_DIM', 'host': '"""127.0.0.1"""', 'port': 'default_server.listen_port', 'collection_name': 'collection_name', 'overwrite': 'create'}), "(dim=EMBEDDING_DIM, host='127.0.0.1', port=default_server.\n    listen_port, collection_name=collection_name, overwrite=create)\n", (2021, 2147), False, 'from llama_index.vector_stores import MilvusVectorStore\n'), ((2237, 2292), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'vector_store': 'vector_store'}), '(vector_store=vector_store)\n', (2265, 2292), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((2333, 2429), 'llama_index.VectorStoreIndex', 'VectorStoreIndex', ([], {'nodes': 'nodes', 'service_context': 'self.service_ctx', 'storage_context': 'storage_ctx'}), '(nodes=nodes, service_context=self.service_ctx,\n    storage_context=storage_ctx)\n', (2349, 2429), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((2588, 2730), 'llama_index.vector_stores.MilvusVectorStore', 'MilvusVectorStore', ([], {'dim': 'EMBEDDING_DIM', 'host': '"""127.0.0.1"""', 'port': 'default_server.listen_port', 'collection_name': 'collection_name', 'overwrite': '(False)'}), "(dim=EMBEDDING_DIM, host='127.0.0.1', port=default_server.\n    listen_port, collection_name=collection_name, overwrite=False)\n", (2605, 2730), False, 'from llama_index.vector_stores import MilvusVectorStore\n'), ((2819, 2874), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'vector_store': 'vector_store'}), '(vector_store=vector_store)\n', (2847, 2874), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((2915, 3043), 'llama_index.VectorStoreIndex.from_vector_store', 'VectorStoreIndex.from_vector_store', ([], {'vector_store': 'vector_store', 'service_context': 'self.service_ctx', 'storage_context': 'storage_ctx'}), '(vector_store=vector_store,\n    service_context=self.service_ctx, storage_context=storage_ctx)\n', (2949, 3043), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((6295, 6327), 'llama_index.node_parser.SimpleNodeParser', 'SimpleNodeParser', ([], {'chunk_size': '(512)'}), '(chunk_size=512)\n', (6311, 6327), False, 'from llama_index.node_parser import SentenceWindowNodeParser, SentenceSplitter, SimpleNodeParser\n'), ((6823, 6966), 'llama_index.vector_stores.MilvusVectorStore', 'MilvusVectorStore', ([], {'dim': 'EMBEDDING_DIM', 'host': '"""127.0.0.1"""', 'port': 'default_server.listen_port', 'collection_name': 'collection_name', 'overwrite': 'create'}), "(dim=EMBEDDING_DIM, host='127.0.0.1', port=default_server.\n    listen_port, collection_name=collection_name, overwrite=create)\n", (6840, 6966), False, 'from llama_index.vector_stores import MilvusVectorStore\n'), ((7056, 7111), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'vector_store': 'vector_store'}), '(vector_store=vector_store)\n', (7084, 7111), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((7152, 7248), 'llama_index.VectorStoreIndex', 'VectorStoreIndex', ([], {'nodes': 'nodes', 'service_context': 'self.service_ctx', 'storage_context': 'storage_ctx'}), '(nodes=nodes, service_context=self.service_ctx,\n    storage_context=storage_ctx)\n', (7168, 7248), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((7407, 7549), 'llama_index.vector_stores.MilvusVectorStore', 'MilvusVectorStore', ([], {'dim': 'EMBEDDING_DIM', 'host': '"""127.0.0.1"""', 'port': 'default_server.listen_port', 'collection_name': 'collection_name', 'overwrite': '(False)'}), "(dim=EMBEDDING_DIM, host='127.0.0.1', port=default_server.\n    listen_port, collection_name=collection_name, overwrite=False)\n", (7424, 7549), False, 'from llama_index.vector_stores import MilvusVectorStore\n'), ((7638, 7693), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'vector_store': 'vector_store'}), '(vector_store=vector_store)\n', (7666, 7693), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((7734, 7862), 'llama_index.VectorStoreIndex.from_vector_store', 'VectorStoreIndex.from_vector_store', ([], {'vector_store': 'vector_store', 'service_context': 'self.service_ctx', 'storage_context': 'storage_ctx'}), '(vector_store=vector_store,\n    service_context=self.service_ctx, storage_context=storage_ctx)\n', (7768, 7862), False, 'from llama_index import ServiceContext, VectorStoreIndex, StorageContext, load_index_from_storage, global_service_context\n'), ((1586, 1671), 'llama_index.readers.SimpleDirectoryReader', 'SimpleDirectoryReader', (['path'], {'file_metadata': 'self.metadata_fn', 'filename_as_id': '(True)'}), '(path, file_metadata=self.metadata_fn, filename_as_id=True\n    )\n', (1607, 1671), False, 'from llama_index.readers import SimpleDirectoryReader\n'), ((5516, 5620), 'llama_index.readers.SimpleDirectoryReader', 'SimpleDirectoryReader', ([], {'input_files': '[file_path]', 'file_metadata': 'self.metadata_fn', 'filename_as_id': '(True)'}), '(input_files=[file_path], file_metadata=self.\n    metadata_fn, filename_as_id=True)\n', (5537, 5620), False, 'from llama_index.readers import SimpleDirectoryReader\n'), ((6254, 6266), 'chatbot.common.path_leaf', 'path_leaf', (['x'], {}), '(x)\n', (6263, 6266), False, 'from chatbot.common import DATA_PATH, EMBEDDING_DIM, EMBEDDING_MODEL, SIMILARITY_SEARCH_THRESHOLD, path_leaf, subjects, PathSep, debug\n'), ((6405, 6490), 'llama_index.readers.SimpleDirectoryReader', 'SimpleDirectoryReader', (['path'], {'file_metadata': 'self.metadata_fn', 'filename_as_id': '(True)'}), '(path, file_metadata=self.metadata_fn, filename_as_id=True\n    )\n', (6426, 6490), False, 'from llama_index.readers import SimpleDirectoryReader\n'), ((3289, 3359), 'llama_index.postprocessor.SimilarityPostprocessor', 'SimilarityPostprocessor', ([], {'similarity_cutoff': 'SIMILARITY_SEARCH_THRESHOLD'}), '(similarity_cutoff=SIMILARITY_SEARCH_THRESHOLD)\n', (3312, 3359), False, 'from llama_index.postprocessor import SimilarityPostprocessor, MetadataReplacementPostProcessor\n'), ((3377, 3439), 'llama_index.postprocessor.MetadataReplacementPostProcessor', 'MetadataReplacementPostProcessor', ([], {'target_metadata_key': '"""window"""'}), "(target_metadata_key='window')\n", (3409, 3439), False, 'from llama_index.postprocessor import SimilarityPostprocessor, MetadataReplacementPostProcessor\n'), ((3721, 3791), 'llama_index.postprocessor.SimilarityPostprocessor', 'SimilarityPostprocessor', ([], {'similarity_cutoff': 'SIMILARITY_SEARCH_THRESHOLD'}), '(similarity_cutoff=SIMILARITY_SEARCH_THRESHOLD)\n', (3744, 3791), False, 'from llama_index.postprocessor import SimilarityPostprocessor, MetadataReplacementPostProcessor\n'), ((3809, 3871), 'llama_index.postprocessor.MetadataReplacementPostProcessor', 'MetadataReplacementPostProcessor', ([], {'target_metadata_key': '"""window"""'}), "(target_metadata_key='window')\n", (3841, 3871), False, 'from llama_index.postprocessor import SimilarityPostprocessor, MetadataReplacementPostProcessor\n'), ((8117, 8179), 'llama_index.postprocessor.MetadataReplacementPostProcessor', 'MetadataReplacementPostProcessor', ([], {'target_metadata_key': '"""window"""'}), "(target_metadata_key='window')\n", (8149, 8179), False, 'from llama_index.postprocessor import SimilarityPostprocessor, MetadataReplacementPostProcessor\n'), ((9294, 9331), 'llama_index.embeddings.HuggingFaceEmbedding', 'HuggingFaceEmbedding', (['EMBEDDING_MODEL'], {}), '(EMBEDDING_MODEL)\n', (9314, 9331), False, 'from llama_index.embeddings import HuggingFaceEmbedding\n')]