code: import lancedb
  import re
  import pickle
  import requests
  import zipfile
  import pandas as pd
  from pathlib import Path
  
  from langchain.document_loaders import BSHTMLLoader
  from langchain.text_splitter import RecursiveCharacterTextSplitter
  from langchain.vectorstores import LanceDB
  
  from sentence_transformers import SentenceTransformer
  model = SentenceTransformer('all-MiniLM-L6-v2')
  
  def get_document_title(document):
      m = str(document.metadata["source"])
      title = re.findall("pandas.documentation(.*).html", m)
      if title[0] is not None:
          return(title[0])
      return ''
  
  def embed_fun(text):
      return [model.encode(x) for x in text]
  
  
  if __name__ == "__main__":
      query = ""
  
      docs_path = Path("docs.pkl")
      docs = []
  
      if not docs_path.exists():
          pandas_docs = requests.get("https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip")
          with open('./tmp/pandas.documentation.zip', 'wb') as f:
              f.write(pandas_docs.content)
  
          file = zipfile.ZipFile("./tmp/pandas.documentation.zip")
          file.extractall(path="./tmp/pandas_docs")
  
          for p in Path("./tmp/pandas_docs/pandas.documentation").rglob("*.html"):
              print(p)
              if p.is_dir():
                  continue
              loader = BSHTMLLoader(p, open_encoding="utf8")
              raw_document = loader.load()
  
              m = {}
              m["title"] = get_document_title(raw_document[0])
              m["version"] = "2.0rc0"
              raw_document[0].metadata = raw_document[0].metadata | m
              raw_document[0].metadata["source"] = str(raw_document[0].metadata["source"])
              docs = docs + raw_document
  
          with docs_path.open("wb") as fh:
              pickle.dump(docs, fh)
      else:
          with docs_path.open("rb") as fh:
              docs = pickle.load(fh)
  
      text_splitter = RecursiveCharacterTextSplitter(
          chunk_size=1000,
          chunk_overlap=200,
      )
      documents = text_splitter.split_documents(docs)
  
      db = lancedb.connect('./tmp/lancedb')
  
      data = [doc.page_content for doc in documents]
      data = pd.DataFrame(data, columns=["text"])
      print(data)
  
      table = db.create_table("pandas_docs", data, embed_fun)
  
      table = db.open_table("pandas_docs")
      print(table.to_pandas())
  
apis: lancedb.connect
extract_api: [((341, 380), 'sentence_transformers.SentenceTransformer', 'SentenceTransformer', (['"""all-MiniLM-L6-v2"""'], {}), "('all-MiniLM-L6-v2')\n", (360, 380), False, 'from sentence_transformers import SentenceTransformer\n'), ((469, 515), 're.findall', 're.findall', (['"""pandas.documentation(.*).html"""', 'm'], {}), "('pandas.documentation(.*).html', m)\n", (479, 515), False, 'import re\n'), ((710, 726), 'pathlib.Path', 'Path', (['"""docs.pkl"""'], {}), "('docs.pkl')\n", (714, 726), False, 'from pathlib import Path\n'), ((1875, 1941), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(1000)', 'chunk_overlap': '(200)'}), '(chunk_size=1000, chunk_overlap=200)\n', (1905, 1941), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((2027, 2059), 'lancedb.connect', 'lancedb.connect', (['"""./tmp/lancedb"""'], {}), "('./tmp/lancedb')\n", (2042, 2059), False, 'import lancedb\n'), ((2123, 2159), 'pandas.DataFrame', 'pd.DataFrame', (['data'], {'columns': "['text']"}), "(data, columns=['text'])\n", (2135, 2159), True, 'import pandas as pd\n'), ((795, 912), 'requests.get', 'requests.get', (['"""https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip"""'], {}), "(\n    'https://eto-public.s3.us-west-2.amazonaws.com/datasets/pandas_docs/pandas.documentation.zip'\n    )\n", (807, 912), False, 'import requests\n'), ((1024, 1073), 'zipfile.ZipFile', 'zipfile.ZipFile', (['"""./tmp/pandas.documentation.zip"""'], {}), "('./tmp/pandas.documentation.zip')\n", (1039, 1073), False, 'import zipfile\n'), ((1300, 1337), 'langchain.document_loaders.BSHTMLLoader', 'BSHTMLLoader', (['p'], {'open_encoding': '"""utf8"""'}), "(p, open_encoding='utf8')\n", (1312, 1337), False, 'from langchain.document_loaders import BSHTMLLoader\n'), ((1746, 1767), 'pickle.dump', 'pickle.dump', (['docs', 'fh'], {}), '(docs, fh)\n', (1757, 1767), False, 'import pickle\n'), ((1838, 1853), 'pickle.load', 'pickle.load', (['fh'], {}), '(fh)\n', (1849, 1853), False, 'import pickle\n'), ((1142, 1188), 'pathlib.Path', 'Path', (['"""./tmp/pandas_docs/pandas.documentation"""'], {}), "('./tmp/pandas_docs/pandas.documentation')\n", (1146, 1188), False, 'from pathlib import Path\n')]