code: from llama_index import SimpleDirectoryReader,VectorStoreIndex  , load_index_from_storage
  from llama_index.storage.storage_context import StorageContext
  from dotenv import load_dotenv
  import logging
  import sys
  
  
  load_dotenv()
  
  # enable INFO level logging
  logging.basicConfig(stream=sys.stdout, level=logging.INFO)
  logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
  
  
  async def load_index(directory_path : str = r'data'):
      documents = SimpleDirectoryReader(directory_path, filename_as_id=True).load_data()
      print(f"loaded documents with {len(documents)} pages")
      try:
          # Rebuild storage context
          storage_context = StorageContext.from_defaults(persist_dir="./storage")
          # Try to load the index from storage
          index = load_index_from_storage(storage_context)
          logging.info("Index loaded from storage.")
      except FileNotFoundError:
  
          logging.info("Index not found. Creating a new one...")
          index = VectorStoreIndex.from_documents(documents)
          # Persist index to disk
          index.storage_context.persist()
          logging.info("New index created and persisted to storage.")
  
      return index
  
  async def update_index(directory_path : str = r'data'):
      try:
          documents = SimpleDirectoryReader(directory_path, filename_as_id=True).load_data()
      except FileNotFoundError:
          logging.error("Invalid document directory path.")
          return None
      try:
          # Rebuild storage context
          storage_context = StorageContext.from_defaults(persist_dir="./storage")
          # Try to load the index from storage
          index = load_index_from_storage(storage_context)
          logging.info("Existing index loaded from storage.")
          refreshed_docs = index.refresh_ref_docs(documents, update_kwargs={"delete_kwargs": {"delete_from_docstore": True}})
          # index.update_ref_doc()
          print(refreshed_docs)
          print('Number of newly inserted/refreshed docs: ', sum(refreshed_docs))
  
          index.storage_context.persist()
          logging.info("Index refreshed and persisted to storage.")
          return refreshed_docs
  
          
      except FileNotFoundError:
      # Run refresh_ref_docs function to check for document updates
          logging.error("Index is not created yet.")
          return None
  
  
apis: llama_index.SimpleDirectoryReader llama_index.VectorStoreIndex.from_documents llama_index.load_index_from_storage llama_index.storage.storage_context.StorageContext.from_defaults
extract_api: [((212, 225), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (223, 225), False, 'from dotenv import load_dotenv\n'), ((255, 313), 'logging.basicConfig', 'logging.basicConfig', ([], {'stream': 'sys.stdout', 'level': 'logging.INFO'}), '(stream=sys.stdout, level=logging.INFO)\n', (274, 313), False, 'import logging\n'), ((345, 385), 'logging.StreamHandler', 'logging.StreamHandler', ([], {'stream': 'sys.stdout'}), '(stream=sys.stdout)\n', (366, 385), False, 'import logging\n'), ((314, 333), 'logging.getLogger', 'logging.getLogger', ([], {}), '()\n', (331, 333), False, 'import logging\n'), ((658, 711), 'llama_index.storage.storage_context.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': '"""./storage"""'}), "(persist_dir='./storage')\n", (686, 711), False, 'from llama_index.storage.storage_context import StorageContext\n'), ((773, 813), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['storage_context'], {}), '(storage_context)\n', (796, 813), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, load_index_from_storage\n'), ((822, 864), 'logging.info', 'logging.info', (['"""Index loaded from storage."""'], {}), "('Index loaded from storage.')\n", (834, 864), False, 'import logging\n'), ((1510, 1563), 'llama_index.storage.storage_context.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': '"""./storage"""'}), "(persist_dir='./storage')\n", (1538, 1563), False, 'from llama_index.storage.storage_context import StorageContext\n'), ((1625, 1665), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['storage_context'], {}), '(storage_context)\n', (1648, 1665), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, load_index_from_storage\n'), ((1674, 1725), 'logging.info', 'logging.info', (['"""Existing index loaded from storage."""'], {}), "('Existing index loaded from storage.')\n", (1686, 1725), False, 'import logging\n'), ((2042, 2099), 'logging.info', 'logging.info', (['"""Index refreshed and persisted to storage."""'], {}), "('Index refreshed and persisted to storage.')\n", (2054, 2099), False, 'import logging\n'), ((459, 517), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', (['directory_path'], {'filename_as_id': '(True)'}), '(directory_path, filename_as_id=True)\n', (480, 517), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, load_index_from_storage\n'), ((904, 958), 'logging.info', 'logging.info', (['"""Index not found. Creating a new one..."""'], {}), "('Index not found. Creating a new one...')\n", (916, 958), False, 'import logging\n'), ((975, 1017), 'llama_index.VectorStoreIndex.from_documents', 'VectorStoreIndex.from_documents', (['documents'], {}), '(documents)\n', (1006, 1017), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, load_index_from_storage\n'), ((1098, 1157), 'logging.info', 'logging.info', (['"""New index created and persisted to storage."""'], {}), "('New index created and persisted to storage.')\n", (1110, 1157), False, 'import logging\n'), ((1371, 1420), 'logging.error', 'logging.error', (['"""Invalid document directory path."""'], {}), "('Invalid document directory path.')\n", (1384, 1420), False, 'import logging\n'), ((2244, 2286), 'logging.error', 'logging.error', (['"""Index is not created yet."""'], {}), "('Index is not created yet.')\n", (2257, 2286), False, 'import logging\n'), ((1262, 1320), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', (['directory_path'], {'filename_as_id': '(True)'}), '(directory_path, filename_as_id=True)\n', (1283, 1320), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, load_index_from_storage\n')]