code: import logging
  from typing import List, Optional
  
  import tiktoken
  from llama_index.indices.base import BaseIndex
  from transformers import AutoTokenizer
  from tiktoken import Encoding
  
  from llama_index import (
      VectorStoreIndex,
      SimpleDirectoryReader,
      ServiceContext,
      StorageContext,
  )
  from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
  from llama_index.node_parser import SimpleNodeParser
  from llama_index import load_index_from_storage
  
  import constants
  from utils import check_index_files
  
  
  class ChunkingTokenizer:
      """Tokenizer for chunking document data for creation of embeddings"""
  
      def __init__(self, model_name: str):
          self.tokenizer = AutoTokenizer.from_pretrained(model_name)
  
      def __call__(self, text: str) -> List[int]:
          return self.tokenizer.encode(text)
  
  
  class Config:
      """Configurations required for initializing the agent"""
  
      _llm_tokenizer: Optional[Encoding] = None
  
      def __init__(
          self,
          encoding_name: str,
          embedding_model_name: str,
      ):
          self._chunking_tokenizer = None
          self.encoding_name = encoding_name
          self.embedding_model_name = embedding_model_name
  
      def initialize(self) -> VectorStoreIndex:
          # Initialize tokenizer for text chunking
          self._chunking_tokenizer = ChunkingTokenizer(self.embedding_model_name)
  
          # Initialize text splitter
          self._text_splitter = TokenTextSplitter(
              chunk_size=constants.MAX_CHUNK_SIZE,
              chunk_overlap=constants.CHUNK_OVERLAP,
              tokenizer=self._chunking_tokenizer,
              separator="\n\n",
              backup_separators=["\n", " "],
          )
  
          # Initialize OpenAI LLM tokenizer
          self.llm_tokenizer = tiktoken.get_encoding(self.encoding_name)
  
          # Initialize vector index
          return self._init_index()
  
      def _init_index(self) -> VectorStoreIndex:
          node_parser = SimpleNodeParser(text_splitter=self._text_splitter)
          service_context = ServiceContext.from_defaults(
              embed_model=f"local:{self.embedding_model_name}", node_parser=node_parser
          )
          index_id = constants.COMPANY_NAME
  
          if check_index_files(constants.PERSIST_DIR):
              storage_context = StorageContext.from_defaults(
                  persist_dir=constants.PERSIST_DIR
              )
              index = load_index_from_storage(
                  storage_context=storage_context,
                  service_context=service_context,
                  index_id=index_id,
              )
  
              return index
  
          # If index does not exist, initialize index
          logging.info('message="initialize index started"')
          # Create index
          documents = SimpleDirectoryReader(constants.DOCUMENT_DATA_DIR).load_data()
          index = VectorStoreIndex.from_documents(
              documents, service_context=service_context
          )
          index.set_index_id(index_id)
          # Save index to disk
          index.storage_context.persist(f"{constants.PERSIST_DIR}")
          logging.info('message="initialize index completed"')
          return index
  
apis: llama_index.VectorStoreIndex.from_documents llama_index.langchain_helpers.text_splitter.TokenTextSplitter llama_index.node_parser.SimpleNodeParser llama_index.ServiceContext.from_defaults llama_index.StorageContext.from_defaults llama_index.SimpleDirectoryReader llama_index.load_index_from_storage
extract_api: [((697, 738), 'transformers.AutoTokenizer.from_pretrained', 'AutoTokenizer.from_pretrained', (['model_name'], {}), '(model_name)\n', (726, 738), False, 'from transformers import AutoTokenizer\n'), ((1440, 1628), 'llama_index.langchain_helpers.text_splitter.TokenTextSplitter', 'TokenTextSplitter', ([], {'chunk_size': 'constants.MAX_CHUNK_SIZE', 'chunk_overlap': 'constants.CHUNK_OVERLAP', 'tokenizer': 'self._chunking_tokenizer', 'separator': '"""\n\n"""', 'backup_separators': "['\\n', ' ']"}), "(chunk_size=constants.MAX_CHUNK_SIZE, chunk_overlap=\n    constants.CHUNK_OVERLAP, tokenizer=self._chunking_tokenizer, separator=\n    '\\n\\n', backup_separators=['\\n', ' '])\n", (1457, 1628), False, 'from llama_index.langchain_helpers.text_splitter import TokenTextSplitter\n'), ((1762, 1803), 'tiktoken.get_encoding', 'tiktoken.get_encoding', (['self.encoding_name'], {}), '(self.encoding_name)\n', (1783, 1803), False, 'import tiktoken\n'), ((1943, 1994), 'llama_index.node_parser.SimpleNodeParser', 'SimpleNodeParser', ([], {'text_splitter': 'self._text_splitter'}), '(text_splitter=self._text_splitter)\n', (1959, 1994), False, 'from llama_index.node_parser import SimpleNodeParser\n'), ((2021, 2129), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'embed_model': 'f"""local:{self.embedding_model_name}"""', 'node_parser': 'node_parser'}), "(embed_model=\n    f'local:{self.embedding_model_name}', node_parser=node_parser)\n", (2049, 2129), False, 'from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext\n'), ((2201, 2241), 'utils.check_index_files', 'check_index_files', (['constants.PERSIST_DIR'], {}), '(constants.PERSIST_DIR)\n', (2218, 2241), False, 'from utils import check_index_files\n'), ((2646, 2696), 'logging.info', 'logging.info', (['"""message="initialize index started\\""""'], {}), '(\'message="initialize index started"\')\n', (2658, 2696), False, 'import logging\n'), ((2819, 2894), 'llama_index.VectorStoreIndex.from_documents', 'VectorStoreIndex.from_documents', (['documents'], {'service_context': 'service_context'}), '(documents, service_context=service_context)\n', (2850, 2894), False, 'from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext\n'), ((3057, 3109), 'logging.info', 'logging.info', (['"""message="initialize index completed\\""""'], {}), '(\'message="initialize index completed"\')\n', (3069, 3109), False, 'import logging\n'), ((2273, 2336), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': 'constants.PERSIST_DIR'}), '(persist_dir=constants.PERSIST_DIR)\n', (2301, 2336), False, 'from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext\n'), ((2387, 2500), 'llama_index.load_index_from_storage', 'load_index_from_storage', ([], {'storage_context': 'storage_context', 'service_context': 'service_context', 'index_id': 'index_id'}), '(storage_context=storage_context, service_context=\n    service_context, index_id=index_id)\n', (2410, 2500), False, 'from llama_index import load_index_from_storage\n'), ((2740, 2790), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', (['constants.DOCUMENT_DATA_DIR'], {}), '(constants.DOCUMENT_DATA_DIR)\n', (2761, 2790), False, 'from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, StorageContext\n')]