code: 
  
  from langchain.text_splitter import (
      RecursiveCharacterTextSplitter,
      Language,
      LatexTextSplitter,
  )
  from langchain.document_loaders import TextLoader
  from langchain.embeddings import OpenAIEmbeddings
  import argparse, os, arxiv
  
  os.environ["OPENAI_API_KEY"] = "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"
  
  embeddings = OpenAIEmbeddings()
  
  from langchain.vectorstores import LanceDB
  from lancedb.pydantic import Vector, LanceModel
  from Typing import List
  from datetime import datetime
  import lancedb
  
  global embedding_out_length
  embedding_out_length = 1536
  class Content(LanceModel):
      id: str
      arxiv_id: str
      vector: Vector(embedding_out_length)
      text: str
      uploaded_date: datetime
      title: str
      authors: List[str]
      abstract: str
      categories: List[str]
      url: str
  
  def PyPDF_to_Vector(table: LanceDB, embeddings: OpenAIEmbeddings, src_dir: str, n_threads: int = 1):
      
      pass    
  
  if __name__ == "__main__":
      argparser = argparse.ArgumentParser(description="Create Vector DB and perform ingestion from source files")
      argparser.add_argument('-s', '--src_dir', type=str, required=True, help = "Source directory where arxiv sources are stored")
      argparser.add_argument('-db', '--db_name', type=str, required=True, help = "Name of the LanceDB database to be created")
      argparser.add_argument('-t', '--table_name', type=str, required=False, help = "Name of the LanceDB table to be created", default = "EIC_archive")
      argparser.add_argument('-openai_key', '--openai_api_key', type=str, required=True, help = "OpenAI API key")
      argparser.add_argument('-c', '--chunking', type = str, required=False, help = "Type of Chunking PDF or LATEX", default = "PDF")
      argparser.add_argument('-n', '--nthreads', type=int, default=-1)
      
      args = argparser.parse_args()
      
      SRC_DIR = args.src_dir
      DB_NAME = args.db_name
      TABLE_NAME = args.table_name
      OPENAI_API_KEY = args.openai_api_key
      NTHREADS = args.nthreads
      
      db = lancedb.connect(DB_NAME)
      table = db.create_table(TABLE_NAME, schema=Content, mode="overwrite")
      
  
  db = lancedb.connect()
  meta_data =  {"arxiv_id": "1", "title": "EIC LLM",
                            "category" : "N/A",
                  "authors": "N/A",
                  "sub_categories": "N/A",
                  "abstract": "N/A",
                  "published": "N/A",
                  "updated": "N/A",
                  "doi": "N/A"
                  },
  table = db.create_table(
      "EIC_archive",
      data=[
          {
              "vector": embeddings.embed_query("EIC LLM"),
              "text": "EIC LLM",
              "id": "1",
              "arxiv_id" : "N/A",
              "title" : "N/A",
              "category" : "N/A",
              "published" : "N/A"
          }
     
      ],
      mode="overwrite",
  )
  
  vectorstore = LanceDB(connection = table, embedding = embeddings)
  
  sourcedir = "PDFs"
  count = 0
  for source in os.listdir(sourcedir):
      if not os.path.isdir(os.path.join("PDFs", source)):
          continue
      print (f"Adding the source document {source} to the Vector DB")
      import arxiv
      client = arxiv.Client()
      search = arxiv.Search(id_list=[source])
      paper = next(arxiv.Client().results(search))
      meta_data = {"arxiv_id": paper.entry_id, 
                  "title": paper.title, 
                  "category" : categories[paper.primary_category],
                  "published": paper.published
                  }
      for file in os.listdir(os.path.join(sourcedir, source)):
          if file.endswith(".tex"):
              latex_file = os.path.join(sourcedir, source, file)
              print (source, latex_file)
              documents = TextLoader(latex_file, encoding = 'latin-1').load()
              latex_splitter = LatexTextSplitter(
                  chunk_size=120, chunk_overlap=10
              )
              documents = latex_splitter.split_documents(documents)
              for doc in documents:
                  for k, v in meta_data.items():
                      doc.metadata[k] = v
              vectorstore.add_documents(documents = documents)
              count+=len(documents)
apis: lancedb.pydantic.Vector lancedb.connect
extract_api: [((342, 360), 'langchain.embeddings.OpenAIEmbeddings', 'OpenAIEmbeddings', ([], {}), '()\n', (358, 360), False, 'from langchain.embeddings import OpenAIEmbeddings\n'), ((2116, 2133), 'lancedb.connect', 'lancedb.connect', ([], {}), '()\n', (2131, 2133), False, 'import lancedb\n'), ((2820, 2867), 'langchain.vectorstores.LanceDB', 'LanceDB', ([], {'connection': 'table', 'embedding': 'embeddings'}), '(connection=table, embedding=embeddings)\n', (2827, 2867), False, 'from langchain.vectorstores import LanceDB\n'), ((2916, 2937), 'os.listdir', 'os.listdir', (['sourcedir'], {}), '(sourcedir)\n', (2926, 2937), False, 'import argparse, os, arxiv\n'), ((648, 676), 'lancedb.pydantic.Vector', 'Vector', (['embedding_out_length'], {}), '(embedding_out_length)\n', (654, 676), False, 'from lancedb.pydantic import Vector, LanceModel\n'), ((978, 1078), 'argparse.ArgumentParser', 'argparse.ArgumentParser', ([], {'description': '"""Create Vector DB and perform ingestion from source files"""'}), "(description=\n    'Create Vector DB and perform ingestion from source files')\n", (1001, 1078), False, 'import argparse, os, arxiv\n'), ((2006, 2030), 'lancedb.connect', 'lancedb.connect', (['DB_NAME'], {}), '(DB_NAME)\n', (2021, 2030), False, 'import lancedb\n'), ((3110, 3124), 'arxiv.Client', 'arxiv.Client', ([], {}), '()\n', (3122, 3124), False, 'import arxiv\n'), ((3138, 3168), 'arxiv.Search', 'arxiv.Search', ([], {'id_list': '[source]'}), '(id_list=[source])\n', (3150, 3168), False, 'import arxiv\n'), ((3458, 3489), 'os.path.join', 'os.path.join', (['sourcedir', 'source'], {}), '(sourcedir, source)\n', (3470, 3489), False, 'import argparse, os, arxiv\n'), ((2964, 2992), 'os.path.join', 'os.path.join', (['"""PDFs"""', 'source'], {}), "('PDFs', source)\n", (2976, 2992), False, 'import argparse, os, arxiv\n'), ((3551, 3588), 'os.path.join', 'os.path.join', (['sourcedir', 'source', 'file'], {}), '(sourcedir, source, file)\n', (3563, 3588), False, 'import argparse, os, arxiv\n'), ((3733, 3784), 'langchain.text_splitter.LatexTextSplitter', 'LatexTextSplitter', ([], {'chunk_size': '(120)', 'chunk_overlap': '(10)'}), '(chunk_size=120, chunk_overlap=10)\n', (3750, 3784), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter, Language, LatexTextSplitter\n'), ((3186, 3200), 'arxiv.Client', 'arxiv.Client', ([], {}), '()\n', (3198, 3200), False, 'import arxiv\n'), ((3652, 3694), 'langchain.document_loaders.TextLoader', 'TextLoader', (['latex_file'], {'encoding': '"""latin-1"""'}), "(latex_file, encoding='latin-1')\n", (3662, 3694), False, 'from langchain.document_loaders import TextLoader\n')]