code: import requests
  import time
  import numpy as np
  import pyarrow as pa
  import lancedb
  import logging
  import os
  
  from tqdm import tqdm
  from pathlib import Path
  from transformers import AutoConfig
  
  
  logging.basicConfig(level=logging.INFO)
  logger = logging.getLogger(__name__)
  
  TEI_URL= os.getenv("EMBED_URL") + "/embed"
  DIRPATH = "/usr/src/docs_dir"
  TABLE_NAME = "docs"
  config = AutoConfig.from_pretrained(os.getenv("EMBED_MODEL"))
  EMB_DIM = config.hidden_size
  CREATE_INDEX = int(os.getenv("CREATE_INDEX"))
  BATCH_SIZE = int(os.getenv("BATCH_SIZE"))
  NUM_PARTITIONS = int(os.getenv("NUM_PARTITIONS"))
  NUM_SUB_VECTORS = int(os.getenv("NUM_SUB_VECTORS"))
  
  HEADERS = {
      "Content-Type": "application/json"
  }
  
  
  def embed_and_index():
      files = Path(DIRPATH).rglob("*")
      texts = []
  
      for file in files:
          if file.is_file():
              try:
                  text = file.open().read()
                  if text:
                      texts.append(text)
              except (OSError, UnicodeDecodeError) as e:
                  logger.error("Error reading file: ", e)
              except Exception as e:
                  logger.error("Unhandled exception: ", e)
                  raise
  
      logger.info(f"Successfully read {len(texts)} files")
  
      db = lancedb.connect("/usr/src/.lancedb")
      schema = pa.schema(
          [
              pa.field("vector", pa.list_(pa.float32(), EMB_DIM)),
              pa.field("text", pa.string()),
          ]
      )
      tbl = db.create_table(TABLE_NAME, schema=schema, mode="overwrite")
  
      start = time.time()
      
      for i in tqdm(range(int(np.ceil(len(texts) / BATCH_SIZE)))):
          payload = {
              "inputs": texts[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
              "truncate": True
          }
  
          resp = requests.post(TEI_URL, json=payload, headers=HEADERS)
          if resp.status_code != 200:
              raise RuntimeError(resp.text)
          vectors = resp.json()
  
          data = [
              {"vector": vec, "text": text}
              for vec, text in zip(vectors, texts[i * BATCH_SIZE:(i + 1) * BATCH_SIZE])
          ]
          tbl.add(data=data)
      
      logger.info(f"Embedding and ingestion of {len(texts)} items took {time.time() - start}")
  
      # IVF-PQ indexing
      if CREATE_INDEX:
          tbl.create_index(num_partitions=NUM_PARTITIONS, num_sub_vectors=NUM_SUB_VECTORS)
  
  
  if __name__ == "__main__":
      embed_and_index()
apis: lancedb.connect
extract_api: [((194, 233), 'logging.basicConfig', 'logging.basicConfig', ([], {'level': 'logging.INFO'}), '(level=logging.INFO)\n', (213, 233), False, 'import logging\n'), ((243, 270), 'logging.getLogger', 'logging.getLogger', (['__name__'], {}), '(__name__)\n', (260, 270), False, 'import logging\n'), ((281, 303), 'os.getenv', 'os.getenv', (['"""EMBED_URL"""'], {}), "('EMBED_URL')\n", (290, 303), False, 'import os\n'), ((401, 425), 'os.getenv', 'os.getenv', (['"""EMBED_MODEL"""'], {}), "('EMBED_MODEL')\n", (410, 425), False, 'import os\n'), ((475, 500), 'os.getenv', 'os.getenv', (['"""CREATE_INDEX"""'], {}), "('CREATE_INDEX')\n", (484, 500), False, 'import os\n'), ((519, 542), 'os.getenv', 'os.getenv', (['"""BATCH_SIZE"""'], {}), "('BATCH_SIZE')\n", (528, 542), False, 'import os\n'), ((565, 592), 'os.getenv', 'os.getenv', (['"""NUM_PARTITIONS"""'], {}), "('NUM_PARTITIONS')\n", (574, 592), False, 'import os\n'), ((616, 644), 'os.getenv', 'os.getenv', (['"""NUM_SUB_VECTORS"""'], {}), "('NUM_SUB_VECTORS')\n", (625, 644), False, 'import os\n'), ((1244, 1280), 'lancedb.connect', 'lancedb.connect', (['"""/usr/src/.lancedb"""'], {}), "('/usr/src/.lancedb')\n", (1259, 1280), False, 'import lancedb\n'), ((1523, 1534), 'time.time', 'time.time', ([], {}), '()\n', (1532, 1534), False, 'import time\n'), ((1746, 1799), 'requests.post', 'requests.post', (['TEI_URL'], {'json': 'payload', 'headers': 'HEADERS'}), '(TEI_URL, json=payload, headers=HEADERS)\n', (1759, 1799), False, 'import requests\n'), ((737, 750), 'pathlib.Path', 'Path', (['DIRPATH'], {}), '(DIRPATH)\n', (741, 750), False, 'from pathlib import Path\n'), ((1409, 1420), 'pyarrow.string', 'pa.string', ([], {}), '()\n', (1418, 1420), True, 'import pyarrow as pa\n'), ((1355, 1367), 'pyarrow.float32', 'pa.float32', ([], {}), '()\n', (1365, 1367), True, 'import pyarrow as pa\n'), ((2166, 2177), 'time.time', 'time.time', ([], {}), '()\n', (2175, 2177), False, 'import time\n')]