code: import os
  import logging
  
  import hashlib
  import PyPDF2
  from tqdm import tqdm
  
  from modules.presets import *
  from modules.utils import *
  from modules.config import local_embedding
  
  
  def get_documents(file_src):
      from langchain.schema import Document
      from langchain.text_splitter import TokenTextSplitter
      text_splitter = TokenTextSplitter(chunk_size=500, chunk_overlap=30)
  
      documents = []
      logging.debug("Loading documents...")
      logging.debug(f"file_src: {file_src}")
      for file in file_src:
          filepath = file.name
          filename = os.path.basename(filepath)
          file_type = os.path.splitext(filename)[1]
          logging.info(f"loading file: {filename}")
          texts = None
          try:
              if file_type == ".pdf":
                  logging.debug("Loading PDF...")
                  try:
                      from modules.pdf_func import parse_pdf
                      from modules.config import advance_docs
  
                      two_column = advance_docs["pdf"].get("two_column", False)
                      pdftext = parse_pdf(filepath, two_column).text
                  except:
                      pdftext = ""
                      with open(filepath, "rb") as pdfFileObj:
                          pdfReader = PyPDF2.PdfReader(pdfFileObj)
                          for page in tqdm(pdfReader.pages):
                              pdftext += page.extract_text()
                  texts = [Document(page_content=pdftext,
                                    metadata={"source": filepath})]
              elif file_type == ".docx":
                  logging.debug("Loading Word...")
                  from langchain.document_loaders import UnstructuredWordDocumentLoader
                  loader = UnstructuredWordDocumentLoader(filepath)
                  texts = loader.load()
              elif file_type == ".pptx":
                  logging.debug("Loading PowerPoint...")
                  from langchain.document_loaders import UnstructuredPowerPointLoader
                  loader = UnstructuredPowerPointLoader(filepath)
                  texts = loader.load()
              elif file_type == ".epub":
                  logging.debug("Loading EPUB...")
                  from langchain.document_loaders import UnstructuredEPubLoader
                  loader = UnstructuredEPubLoader(filepath)
                  texts = loader.load()
              elif file_type == ".xlsx":
                  logging.debug("Loading Excel...")
                  text_list = excel_to_string(filepath)
                  texts = []
                  for elem in text_list:
                      texts.append(Document(page_content=elem,
                                   metadata={"source": filepath}))
              elif file_type in [".jpg", ".jpeg", ".png", ".heif", ".heic", ".webp", ".bmp", ".gif", ".tiff", ".tif"]:
                  raise gr.Warning(i18n("不支持的文件: ") + filename + i18n("，请使用 .pdf, .docx, .pptx, .epub, .xlsx 等文档。"))
              else:
                  logging.debug("Loading text file...")
                  from langchain.document_loaders import TextLoader
                  loader = TextLoader(filepath, "utf8")
                  texts = loader.load()
          except Exception as e:
              import traceback
              logging.error(f"Error loading file: {filename}")
              traceback.print_exc()
  
          if texts is not None:
              texts = text_splitter.split_documents(texts)
              documents.extend(texts)
      logging.debug("Documents loaded.")
      return documents
  
  
  def construct_index(
      api_key,
      file_src,
      max_input_size=4096,
      num_outputs=5,
      max_chunk_overlap=20,
      chunk_size_limit=600,
      embedding_limit=None,
      separator=" ",
      load_from_cache_if_possible=True,
  ):
      from langchain.chat_models import ChatOpenAI
      from langchain.vectorstores import FAISS
  
      if api_key:
          os.environ["OPENAI_API_KEY"] = api_key
      else:
          # 由于一个依赖的愚蠢的设计，这里必须要有一个API KEY
          os.environ["OPENAI_API_KEY"] = "sk-xxxxxxx"
      logging.debug(f"api base: {os.environ.get('OPENAI_API_BASE', None)}")
      chunk_size_limit = None if chunk_size_limit == 0 else chunk_size_limit
      embedding_limit = None if embedding_limit == 0 else embedding_limit
      separator = " " if separator == "" else separator
  
      index_name = get_file_hash(file_src)
      index_path = f"./index/{index_name}"
      if local_embedding:
          from langchain.embeddings.huggingface import HuggingFaceEmbeddings
          embeddings = HuggingFaceEmbeddings(
              model_name="sentence-transformers/distiluse-base-multilingual-cased-v2")
      else:
          from langchain.embeddings import OpenAIEmbeddings
          if os.environ.get("OPENAI_API_TYPE", "openai") == "openai":
              embeddings = OpenAIEmbeddings(openai_api_base=os.environ.get(
                  "OPENAI_API_BASE", None), openai_api_key=os.environ.get("OPENAI_EMBEDDING_API_KEY", api_key))
          else:
              embeddings = OpenAIEmbeddings(deployment=os.environ["AZURE_EMBEDDING_DEPLOYMENT_NAME"], openai_api_key=os.environ["AZURE_OPENAI_API_KEY"],
                                            model=os.environ["AZURE_EMBEDDING_MODEL_NAME"], openai_api_base=os.environ["AZURE_OPENAI_API_BASE_URL"], openai_api_type="azure")
      if os.path.exists(index_path) and load_from_cache_if_possible:
          logging.info(i18n("找到了缓存的索引文件，加载中……"))
          return FAISS.load_local(index_path, embeddings)
      else:
          documents = get_documents(file_src)
          logging.debug(i18n("构建索引中……"))
          if documents:
              with retrieve_proxy():
                  index = FAISS.from_documents(documents, embeddings)
          else:
              raise Exception(i18n("没有找到任何支持的文档。"))
          logging.debug(i18n("索引构建完成！"))
          os.makedirs("./index", exist_ok=True)
          index.save_local(index_path)
          logging.debug(i18n("索引已保存至本地!"))
          return index
  
apis: langchain.embeddings.huggingface.HuggingFaceEmbeddings langchain.document_loaders.UnstructuredWordDocumentLoader langchain.vectorstores.FAISS.from_documents langchain.document_loaders.UnstructuredPowerPointLoader langchain.schema.Document langchain.embeddings.OpenAIEmbeddings langchain.vectorstores.FAISS.load_local langchain.document_loaders.TextLoader langchain.document_loaders.UnstructuredEPubLoader langchain.text_splitter.TokenTextSplitter
extract_api: [((330, 381), 'langchain.text_splitter.TokenTextSplitter', 'TokenTextSplitter', ([], {'chunk_size': '(500)', 'chunk_overlap': '(30)'}), '(chunk_size=500, chunk_overlap=30)\n', (347, 381), False, 'from langchain.text_splitter import TokenTextSplitter\n'), ((406, 443), 'logging.debug', 'logging.debug', (['"""Loading documents..."""'], {}), "('Loading documents...')\n", (419, 443), False, 'import logging\n'), ((448, 486), 'logging.debug', 'logging.debug', (['f"""file_src: {file_src}"""'], {}), "(f'file_src: {file_src}')\n", (461, 486), False, 'import logging\n'), ((3415, 3449), 'logging.debug', 'logging.debug', (['"""Documents loaded."""'], {}), "('Documents loaded.')\n", (3428, 3449), False, 'import logging\n'), ((561, 587), 'os.path.basename', 'os.path.basename', (['filepath'], {}), '(filepath)\n', (577, 587), False, 'import os\n'), ((646, 687), 'logging.info', 'logging.info', (['f"""loading file: {filename}"""'], {}), "(f'loading file: {filename}')\n", (658, 687), False, 'import logging\n'), ((4440, 4539), 'langchain.embeddings.huggingface.HuggingFaceEmbeddings', 'HuggingFaceEmbeddings', ([], {'model_name': '"""sentence-transformers/distiluse-base-multilingual-cased-v2"""'}), "(model_name=\n    'sentence-transformers/distiluse-base-multilingual-cased-v2')\n", (4461, 4539), False, 'from langchain.embeddings.huggingface import HuggingFaceEmbeddings\n'), ((5212, 5238), 'os.path.exists', 'os.path.exists', (['index_path'], {}), '(index_path)\n', (5226, 5238), False, 'import os\n'), ((5334, 5374), 'langchain.vectorstores.FAISS.load_local', 'FAISS.load_local', (['index_path', 'embeddings'], {}), '(index_path, embeddings)\n', (5350, 5374), False, 'from langchain.vectorstores import FAISS\n'), ((5704, 5741), 'os.makedirs', 'os.makedirs', (['"""./index"""'], {'exist_ok': '(True)'}), "('./index', exist_ok=True)\n", (5715, 5741), False, 'import os\n'), ((608, 634), 'os.path.splitext', 'os.path.splitext', (['filename'], {}), '(filename)\n', (624, 634), False, 'import os\n'), ((4627, 4670), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_TYPE"""', '"""openai"""'], {}), "('OPENAI_API_TYPE', 'openai')\n", (4641, 4670), False, 'import os\n'), ((4907, 5176), 'langchain.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {'deployment': "os.environ['AZURE_EMBEDDING_DEPLOYMENT_NAME']", 'openai_api_key': "os.environ['AZURE_OPENAI_API_KEY']", 'model': "os.environ['AZURE_EMBEDDING_MODEL_NAME']", 'openai_api_base': "os.environ['AZURE_OPENAI_API_BASE_URL']", 'openai_api_type': '"""azure"""'}), "(deployment=os.environ['AZURE_EMBEDDING_DEPLOYMENT_NAME'],\n    openai_api_key=os.environ['AZURE_OPENAI_API_KEY'], model=os.environ[\n    'AZURE_EMBEDDING_MODEL_NAME'], openai_api_base=os.environ[\n    'AZURE_OPENAI_API_BASE_URL'], openai_api_type='azure')\n", (4923, 5176), False, 'from langchain.embeddings import OpenAIEmbeddings\n'), ((774, 805), 'logging.debug', 'logging.debug', (['"""Loading PDF..."""'], {}), "('Loading PDF...')\n", (787, 805), False, 'import logging\n'), ((3204, 3252), 'logging.error', 'logging.error', (['f"""Error loading file: {filename}"""'], {}), "(f'Error loading file: {filename}')\n", (3217, 3252), False, 'import logging\n'), ((3265, 3286), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (3284, 3286), False, 'import traceback\n'), ((3993, 4032), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_BASE"""', 'None'], {}), "('OPENAI_API_BASE', None)\n", (4007, 4032), False, 'import os\n'), ((5549, 5592), 'langchain.vectorstores.FAISS.from_documents', 'FAISS.from_documents', (['documents', 'embeddings'], {}), '(documents, embeddings)\n', (5569, 5592), False, 'from langchain.vectorstores import FAISS\n'), ((1418, 1479), 'langchain.schema.Document', 'Document', ([], {'page_content': 'pdftext', 'metadata': "{'source': filepath}"}), "(page_content=pdftext, metadata={'source': filepath})\n", (1426, 1479), False, 'from langchain.schema import Document\n'), ((1570, 1602), 'logging.debug', 'logging.debug', (['"""Loading Word..."""'], {}), "('Loading Word...')\n", (1583, 1602), False, 'import logging\n'), ((1714, 1754), 'langchain.document_loaders.UnstructuredWordDocumentLoader', 'UnstructuredWordDocumentLoader', (['filepath'], {}), '(filepath)\n', (1744, 1754), False, 'from langchain.document_loaders import UnstructuredWordDocumentLoader\n'), ((4742, 4781), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_BASE"""', 'None'], {}), "('OPENAI_API_BASE', None)\n", (4756, 4781), False, 'import os\n'), ((4815, 4866), 'os.environ.get', 'os.environ.get', (['"""OPENAI_EMBEDDING_API_KEY"""', 'api_key'], {}), "('OPENAI_EMBEDDING_API_KEY', api_key)\n", (4829, 4866), False, 'import os\n'), ((1055, 1086), 'modules.pdf_func.parse_pdf', 'parse_pdf', (['filepath', 'two_column'], {}), '(filepath, two_column)\n', (1064, 1086), False, 'from modules.pdf_func import parse_pdf\n'), ((1848, 1886), 'logging.debug', 'logging.debug', (['"""Loading PowerPoint..."""'], {}), "('Loading PowerPoint...')\n", (1861, 1886), False, 'import logging\n'), ((1996, 2034), 'langchain.document_loaders.UnstructuredPowerPointLoader', 'UnstructuredPowerPointLoader', (['filepath'], {}), '(filepath)\n', (2024, 2034), False, 'from langchain.document_loaders import UnstructuredPowerPointLoader\n'), ((1246, 1274), 'PyPDF2.PdfReader', 'PyPDF2.PdfReader', (['pdfFileObj'], {}), '(pdfFileObj)\n', (1262, 1274), False, 'import PyPDF2\n'), ((1311, 1332), 'tqdm.tqdm', 'tqdm', (['pdfReader.pages'], {}), '(pdfReader.pages)\n', (1315, 1332), False, 'from tqdm import tqdm\n'), ((2128, 2160), 'logging.debug', 'logging.debug', (['"""Loading EPUB..."""'], {}), "('Loading EPUB...')\n", (2141, 2160), False, 'import logging\n'), ((2264, 2296), 'langchain.document_loaders.UnstructuredEPubLoader', 'UnstructuredEPubLoader', (['filepath'], {}), '(filepath)\n', (2286, 2296), False, 'from langchain.document_loaders import UnstructuredEPubLoader\n'), ((2390, 2423), 'logging.debug', 'logging.debug', (['"""Loading Excel..."""'], {}), "('Loading Excel...')\n", (2403, 2423), False, 'import logging\n'), ((2936, 2973), 'logging.debug', 'logging.debug', (['"""Loading text file..."""'], {}), "('Loading text file...')\n", (2949, 2973), False, 'import logging\n'), ((3065, 3093), 'langchain.document_loaders.TextLoader', 'TextLoader', (['filepath', '"""utf8"""'], {}), "(filepath, 'utf8')\n", (3075, 3093), False, 'from langchain.document_loaders import TextLoader\n'), ((2577, 2635), 'langchain.schema.Document', 'Document', ([], {'page_content': 'elem', 'metadata': "{'source': filepath}"}), "(page_content=elem, metadata={'source': filepath})\n", (2585, 2635), False, 'from langchain.schema import Document\n')]