code: import os
  import shutil
  from pathlib import Path
  
  import lancedb
  from lancedb.pydantic import LanceModel, Vector, pydantic_to_schema
  from langchain.document_loaders import TextLoader
  from langchain.embeddings import HuggingFaceEmbeddings
  from langchain.text_splitter import RecursiveCharacterTextSplitter
  from langchain.vectorstores import LanceDB
  
  
  # LanceDB pydantic schema
  class Content(LanceModel):
      text: str
      vector: Vector(384)
  
  
  def get_files() -> list[str]:
      # Get a list of files from the data directory
      data_dir = Path("../data")
      txt_files = list(data_dir.glob("*.txt"))
      # Return string of paths or else lancedb/pydantic will complain
      txt_files = [str(f) for f in txt_files]
      return txt_files
  
  
  def get_docs(txt_files: list[str]):
      loaders = [TextLoader(f) for f in txt_files]
      docs = [loader.load() for loader in loaders]
      return docs
  
  
  def create_lance_table(table_name: str) -> lancedb.table.LanceTable:
      try:
          # Create empty table if it does not exist
          tbl = db.create_table(table_name, schema=pydantic_to_schema(Content), mode="overwrite")
      except OSError:
          # If table exists, open it
          tbl = db.open_table(table_name, mode="append")
      return tbl
  
  
  async def search_lancedb(query: str, retriever: LanceDB) -> list[Content]:
      "Perform async retrieval from LanceDB"
      search_result = await retriever.asimilarity_search(query, k=5)
      if len(search_result) > 0:
          print(search_result[0].page_content)
      else:
          print("Failed to find similar result")
      return search_result
  
  
  def main() -> None:
      txt_files = get_files()
      text_splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=50)
      embeddings = HuggingFaceEmbeddings(
          model_name="sentence-transformers/all-MiniLM-L6-v2", model_kwargs={"device": "cpu"}
      )
      tbl = create_lance_table("countries")
      docs = get_docs(txt_files)
      chunked_docs = []
      for doc in docs:
          chunked_docs.extend(text_splitter.split_documents(doc))
      # Ingest docs in append mode
      retriever = LanceDB.from_documents(chunked_docs, embeddings, connection=tbl)
      return retriever
  
  
  if __name__ == "__main__":
      DB_NAME = "./db"
      TABLE = "countries"
      if os.path.exists(DB_NAME):
          # Clear DB if it exists
          shutil.rmtree(DB_NAME)
  
      db = lancedb.connect(DB_NAME)
      retriever = main()
      print("Finished loading documents to LanceDB")
  
      query = "Is Tonga a monarchy or a democracy"
      docsearch = retriever.as_retriever(
          search_kwargs={"k": 3, "threshold": 0.8, "return_vector": False}
      )
  
      search_result = docsearch.get_relevant_documents(query)
  
      if len(search_result) > 0:
          print(f"Found {len(search_result)} relevant results")
          print([r.page_content for r in search_result])
      else:
          print("Failed to find relevant result")
apis: lancedb.pydantic.Vector lancedb.connect lancedb.pydantic.pydantic_to_schema
extract_api: [((429, 440), 'lancedb.pydantic.Vector', 'Vector', (['(384)'], {}), '(384)\n', (435, 440), False, 'from lancedb.pydantic import LanceModel, Vector, pydantic_to_schema\n'), ((538, 553), 'pathlib.Path', 'Path', (['"""../data"""'], {}), "('../data')\n", (542, 553), False, 'from pathlib import Path\n'), ((1650, 1714), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(512)', 'chunk_overlap': '(50)'}), '(chunk_size=512, chunk_overlap=50)\n', (1680, 1714), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1732, 1842), 'langchain.embeddings.HuggingFaceEmbeddings', 'HuggingFaceEmbeddings', ([], {'model_name': '"""sentence-transformers/all-MiniLM-L6-v2"""', 'model_kwargs': "{'device': 'cpu'}"}), "(model_name='sentence-transformers/all-MiniLM-L6-v2',\n    model_kwargs={'device': 'cpu'})\n", (1753, 1842), False, 'from langchain.embeddings import HuggingFaceEmbeddings\n'), ((2082, 2146), 'langchain.vectorstores.LanceDB.from_documents', 'LanceDB.from_documents', (['chunked_docs', 'embeddings'], {'connection': 'tbl'}), '(chunked_docs, embeddings, connection=tbl)\n', (2104, 2146), False, 'from langchain.vectorstores import LanceDB\n'), ((2249, 2272), 'os.path.exists', 'os.path.exists', (['DB_NAME'], {}), '(DB_NAME)\n', (2263, 2272), False, 'import os\n'), ((2347, 2371), 'lancedb.connect', 'lancedb.connect', (['DB_NAME'], {}), '(DB_NAME)\n', (2362, 2371), False, 'import lancedb\n'), ((785, 798), 'langchain.document_loaders.TextLoader', 'TextLoader', (['f'], {}), '(f)\n', (795, 798), False, 'from langchain.document_loaders import TextLoader\n'), ((2314, 2336), 'shutil.rmtree', 'shutil.rmtree', (['DB_NAME'], {}), '(DB_NAME)\n', (2327, 2336), False, 'import shutil\n'), ((1063, 1090), 'lancedb.pydantic.pydantic_to_schema', 'pydantic_to_schema', (['Content'], {}), '(Content)\n', (1081, 1090), False, 'from lancedb.pydantic import LanceModel, Vector, pydantic_to_schema\n')]