code: from FlagEmbedding import LLMEmbedder, FlagReranker
  import lancedb
  import re
  import pandas as pd
  import random
  
  from datasets import load_dataset
  
  import torch
  import gc
  
  from lancedb.embeddings import with_embeddings
  
  embed_model = LLMEmbedder(
      "BAAI/llm-embedder", use_fp16=False
  )  # Load model (automatically use GPUs)
  reranker_model = FlagReranker(
      "BAAI/bge-reranker-base", use_fp16=True
  )  # use_fp16 speeds up computation with a slight performance degradation
  task = "qa"  # Encode for a specific task (qa, icl, chat, lrlm, tool, convsearch)
  
  
  # get embedding using LLM embedder
  def embed_documents(batch):
      """
      Function to embed the whole text data
      """
      return embed_model.encode_keys(batch, task=task)  # Encode data or 'keys'
  
  
  def search(table, query, top_k=10):
      """
      Search a query from the table
      """
      query_vector = embed_model.encode_queries(
          query, task=task
      )  # Encode the QUERY (it is done differently than the 'key')
      search_results = table.search(query_vector).limit(top_k)
      return search_results
  
  
  def rerank(query, search_results):
      search_results["old_similarity_rank"] = search_results.index + 1  # Old ranks
  
      torch.cuda.empty_cache()
      gc.collect()
  
      search_results["new_scores"] = reranker_model.compute_score(
          [[query, chunk] for chunk in search_results["text"]]
      )  # Re compute ranks
      return search_results.sort_values(by="new_scores", ascending=False).reset_index(
          drop=True
      )
  
  
  def main():
      queries = load_dataset("BeIR/scidocs", "queries")["queries"].to_pandas()
      docs = (
          load_dataset("BeIR/scidocs", "corpus")["corpus"]
          .to_pandas()
          .dropna(subset="text")
          .sample(10000)
      )  # just random samples for faster embed demo
  
      # create Database using LanceDB Cloud
      uri = "db://your-project-slug"
      api_key = "sk_..."
      db = lancedb.connect(uri, api_key=api_key, region="us-east-1")
      table_name = "doc_embed"
      try:
          # Use the train text chunk data to save embed in the DB
          data = with_embeddings(
              embed_documents, docs, column="text", show_progress=True, batch_size=128
          )
          table = db.create_table(table_name, data=data)  # create Table
      except:
          table = db.open_table(table_name)  # Open Table
  
          query = random.choice(queries["text"])
          print("QUERY:-> ", query)
  
          # get top_k search results
          search_results = (
              search(table, "what is mitochondria?", top_k=10)
              .to_pandas()
              .dropna(subset="text")
              .reset_index(drop=True)
          )
          print("SEARCH RESULTS:-> ", search_results)
  
          # Rerank search results using Reranker from BGE Reranker
          print("QUERY:-> ", query)
          search_results_reranked = rerank(query, search_results)
          print("SEARCH RESULTS RERANKED:-> ", search_results_reranked)
  
  
  if __name__ == "__main__":
      main()
  
apis: lancedb.embeddings.with_embeddings lancedb.connect
extract_api: [((233, 281), 'FlagEmbedding.LLMEmbedder', 'LLMEmbedder', (['"""BAAI/llm-embedder"""'], {'use_fp16': '(False)'}), "('BAAI/llm-embedder', use_fp16=False)\n", (244, 281), False, 'from FlagEmbedding import LLMEmbedder, FlagReranker\n'), ((344, 397), 'FlagEmbedding.FlagReranker', 'FlagReranker', (['"""BAAI/bge-reranker-base"""'], {'use_fp16': '(True)'}), "('BAAI/bge-reranker-base', use_fp16=True)\n", (356, 397), False, 'from FlagEmbedding import LLMEmbedder, FlagReranker\n'), ((1196, 1220), 'torch.cuda.empty_cache', 'torch.cuda.empty_cache', ([], {}), '()\n', (1218, 1220), False, 'import torch\n'), ((1225, 1237), 'gc.collect', 'gc.collect', ([], {}), '()\n', (1235, 1237), False, 'import gc\n'), ((1897, 1954), 'lancedb.connect', 'lancedb.connect', (['uri'], {'api_key': 'api_key', 'region': '"""us-east-1"""'}), "(uri, api_key=api_key, region='us-east-1')\n", (1912, 1954), False, 'import lancedb\n'), ((2072, 2165), 'lancedb.embeddings.with_embeddings', 'with_embeddings', (['embed_documents', 'docs'], {'column': '"""text"""', 'show_progress': '(True)', 'batch_size': '(128)'}), "(embed_documents, docs, column='text', show_progress=True,\n    batch_size=128)\n", (2087, 2165), False, 'from lancedb.embeddings import with_embeddings\n'), ((2340, 2370), 'random.choice', 'random.choice', (["queries['text']"], {}), "(queries['text'])\n", (2353, 2370), False, 'import random\n'), ((1528, 1567), 'datasets.load_dataset', 'load_dataset', (['"""BeIR/scidocs"""', '"""queries"""'], {}), "('BeIR/scidocs', 'queries')\n", (1540, 1567), False, 'from datasets import load_dataset\n'), ((1612, 1650), 'datasets.load_dataset', 'load_dataset', (['"""BeIR/scidocs"""', '"""corpus"""'], {}), "('BeIR/scidocs', 'corpus')\n", (1624, 1650), False, 'from datasets import load_dataset\n')]