code: from glob import glob
  from os.path import basename
  from pathlib import Path
  
  import chromadb
  import lancedb
  import pandas as pd
  import torch
  from chromadb.utils import embedding_functions
  from lancedb.embeddings import EmbeddingFunctionRegistry
  from lancedb.pydantic import LanceModel, Vector
  from loguru import logger
  from rich import print
  from rich.progress import track
  
  MODEL_NAME = "all-distilroberta-v1"
  DB_PATH = "db/lancedb-test"
  TABLE_NAME = COLLECTION_NAME = "test"
  
  registry = EmbeddingFunctionRegistry.get_instance()
  func = registry.get("sentence-transformers").create(
      name="all-distilroberta-v1", device="cuda" if torch.cuda.is_available() else "cpu"
  )
  
  
  class Document(LanceModel):
      document: str = func.SourceField()
      embedding: Vector(func.ndims()) = func.VectorField()
      source: str
  
  
  def get_collection() -> chromadb.Collection:
      chroma_client = chromadb.PersistentClient(DB_PATH)
  
      try:
          collection = chroma_client.get_collection(name=COLLECTION_NAME)
      except Exception as e:
          logger.exception(e)
          logger.warning("Indexing documents...")
          collection = chroma_client.create_collection(name=COLLECTION_NAME)
          csvs = glob("crawled/*.csv")
          sentence_transformer_ef = (
              embedding_functions.SentenceTransformerEmbeddingFunction(
                  model_name=MODEL_NAME
              )
          )
          data = []
          for csv in track(csvs):
              df = pd.read_csv(csv)
              if len(df) == 0:
                  continue
              urls, documents = df["URL"].tolist(), df["Section Content"].tolist()
              embeddings = sentence_transformer_ef(documents)
              assert len(urls) == len(documents) == len(embeddings)
              base = basename(urls[0])
              collection.add(
                  embeddings=embeddings,
                  documents=documents,
                  metadatas=[{"source": url} for url in urls],
                  ids=[f"{base}_{i}" for i in range(len(documents))],
              )
  
      return collection
  
  
  def get_table():
      uri = DB_PATH[:]
      db = lancedb.connect(uri)
      table = db.open_table(TABLE_NAME)
      return table
  
apis: lancedb.connect lancedb.embeddings.EmbeddingFunctionRegistry.get_instance
extract_api: [((489, 529), 'lancedb.embeddings.EmbeddingFunctionRegistry.get_instance', 'EmbeddingFunctionRegistry.get_instance', ([], {}), '()\n', (527, 529), False, 'from lancedb.embeddings import EmbeddingFunctionRegistry\n'), ((881, 915), 'chromadb.PersistentClient', 'chromadb.PersistentClient', (['DB_PATH'], {}), '(DB_PATH)\n', (906, 915), False, 'import chromadb\n'), ((2082, 2102), 'lancedb.connect', 'lancedb.connect', (['uri'], {}), '(uri)\n', (2097, 2102), False, 'import lancedb\n'), ((633, 658), 'torch.cuda.is_available', 'torch.cuda.is_available', ([], {}), '()\n', (656, 658), False, 'import torch\n'), ((1033, 1052), 'loguru.logger.exception', 'logger.exception', (['e'], {}), '(e)\n', (1049, 1052), False, 'from loguru import logger\n'), ((1061, 1100), 'loguru.logger.warning', 'logger.warning', (['"""Indexing documents..."""'], {}), "('Indexing documents...')\n", (1075, 1100), False, 'from loguru import logger\n'), ((1191, 1212), 'glob.glob', 'glob', (['"""crawled/*.csv"""'], {}), "('crawled/*.csv')\n", (1195, 1212), False, 'from glob import glob\n'), ((1261, 1340), 'chromadb.utils.embedding_functions.SentenceTransformerEmbeddingFunction', 'embedding_functions.SentenceTransformerEmbeddingFunction', ([], {'model_name': 'MODEL_NAME'}), '(model_name=MODEL_NAME)\n', (1317, 1340), False, 'from chromadb.utils import embedding_functions\n'), ((1418, 1429), 'rich.progress.track', 'track', (['csvs'], {}), '(csvs)\n', (1423, 1429), False, 'from rich.progress import track\n'), ((1448, 1464), 'pandas.read_csv', 'pd.read_csv', (['csv'], {}), '(csv)\n', (1459, 1464), True, 'import pandas as pd\n'), ((1745, 1762), 'os.path.basename', 'basename', (['urls[0]'], {}), '(urls[0])\n', (1753, 1762), False, 'from os.path import basename\n')]