code: #!/usr/bin/env python
  # -*- encoding: utf-8 -*-
  '''
  @File    :   create_db.py
  @Time    :   2023/12/14 10:56:31
  @Author  :   Logan Zou 
  @Version :   1.0
  @Contact :   loganzou0421@163.com
  @License :   (C)Copyright 2017-2018, Liugroup-NLPR-CASIA
  @Desc    :   知识库搭建
  '''
  
  # 首先导入所需第三方库
  from langchain.document_loaders import UnstructuredFileLoader
  from langchain.document_loaders import UnstructuredMarkdownLoader
  from langchain.text_splitter import RecursiveCharacterTextSplitter
  from langchain.vectorstores import Chroma
  from langchain.embeddings.huggingface import HuggingFaceEmbeddings
  from tqdm import tqdm
  import os
  
  # 获取文件路径函数
  def get_files(dir_path):
      # args：dir_path，目标文件夹路径
      file_list = []
      for filepath, dirnames, filenames in os.walk(dir_path):
          # os.walk 函数将递归遍历指定文件夹
          for filename in filenames:
              # 通过后缀名判断文件类型是否满足要求
              if filename.endswith(".md"):
                  # 如果满足要求，将其绝对路径加入到结果列表
                  file_list.append(os.path.join(filepath, filename))
              elif filename.endswith(".txt"):
                  file_list.append(os.path.join(filepath, filename))
      return file_list
  
  # 加载文件函数
  def get_text(dir_path):
      # args：dir_path，目标文件夹路径
      # 首先调用上文定义的函数得到目标文件路径列表
      file_lst = get_files(dir_path)
      # docs 存放加载之后的纯文本对象
      docs = []
      # 遍历所有目标文件
      for one_file in tqdm(file_lst):
          file_type = one_file.split('.')[-1]
          if file_type == 'md':
              loader = UnstructuredMarkdownLoader(one_file)
          elif file_type == 'txt':
              loader = UnstructuredFileLoader(one_file)
          else:
              # 如果是不符合条件的文件，直接跳过
              continue
          docs.extend(loader.load())
      return docs
  
  # 目标文件夹
  tar_dir = [
      "/root/autodl-tmp/self-llm",
      "/root/autodl-tmp/llm-universe",
      "/root/autodl-tmp/prompt-engineering-for-developers",
      "/root/autodl-tmp/so-large-lm",
      "/root/autodl-tmp/hugging-llm",
  ]
  
  # 加载目标文件
  docs = []
  for dir_path in tar_dir:
      docs.extend(get_text(dir_path))
  
  # 对文本进行分块
  text_splitter = RecursiveCharacterTextSplitter(
      chunk_size=500, chunk_overlap=150)
  split_docs = text_splitter.split_documents(docs)
  
  # 加载开源词向量模型
  embeddings = HuggingFaceEmbeddings(model_name="/root/autodl-tmp/sentence-transformer")
  
  # 构建向量数据库
  # 定义持久化路径
  persist_directory = 'data_base/vector_db/chroma'
  # 加载数据库
  vectordb = Chroma.from_documents(
      documents=split_docs,
      embedding=embeddings,
      persist_directory=persist_directory  # 允许我们将persist_directory目录保存到磁盘上
  )
  # 将加载的向量数据库持久化到磁盘上
  vectordb.persist()
apis: langchain.embeddings.huggingface.HuggingFaceEmbeddings langchain.vectorstores.Chroma.from_documents langchain.text_splitter.RecursiveCharacterTextSplitter langchain.document_loaders.UnstructuredFileLoader langchain.document_loaders.UnstructuredMarkdownLoader
extract_api: [((2018, 2083), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(500)', 'chunk_overlap': '(150)'}), '(chunk_size=500, chunk_overlap=150)\n', (2048, 2083), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((2164, 2237), 'langchain.embeddings.huggingface.HuggingFaceEmbeddings', 'HuggingFaceEmbeddings', ([], {'model_name': '"""/root/autodl-tmp/sentence-transformer"""'}), "(model_name='/root/autodl-tmp/sentence-transformer')\n", (2185, 2237), False, 'from langchain.embeddings.huggingface import HuggingFaceEmbeddings\n'), ((2327, 2433), 'langchain.vectorstores.Chroma.from_documents', 'Chroma.from_documents', ([], {'documents': 'split_docs', 'embedding': 'embeddings', 'persist_directory': 'persist_directory'}), '(documents=split_docs, embedding=embeddings,\n    persist_directory=persist_directory)\n', (2348, 2433), False, 'from langchain.vectorstores import Chroma\n'), ((741, 758), 'os.walk', 'os.walk', (['dir_path'], {}), '(dir_path)\n', (748, 758), False, 'import os\n'), ((1335, 1349), 'tqdm.tqdm', 'tqdm', (['file_lst'], {}), '(file_lst)\n', (1339, 1349), False, 'from tqdm import tqdm\n'), ((1446, 1482), 'langchain.document_loaders.UnstructuredMarkdownLoader', 'UnstructuredMarkdownLoader', (['one_file'], {}), '(one_file)\n', (1472, 1482), False, 'from langchain.document_loaders import UnstructuredMarkdownLoader\n'), ((1537, 1569), 'langchain.document_loaders.UnstructuredFileLoader', 'UnstructuredFileLoader', (['one_file'], {}), '(one_file)\n', (1559, 1569), False, 'from langchain.document_loaders import UnstructuredFileLoader\n'), ((971, 1003), 'os.path.join', 'os.path.join', (['filepath', 'filename'], {}), '(filepath, filename)\n', (983, 1003), False, 'import os\n'), ((1082, 1114), 'os.path.join', 'os.path.join', (['filepath', 'filename'], {}), '(filepath, filename)\n', (1094, 1114), False, 'import os\n')]