code: from dotenv import load_dotenv
  import os
  import lancedb
  import torch
  from PIL import Image
  import glob
  import re
  
  from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
  import concurrent.futures
  
  # Set options for youtube_dl
  ydl_opts = {
      "quiet": True,  # Silence youtube_dl output
      "extract_flat": True,  # Extract metadata only, no download
  }
  
  MODEL_ID = None
  MODEL = None
  TOKENIZER = None
  PROCESSOR = None
  
  
  def setup_clip_model(model_id):
      global MODEL_ID, MODEL, TOKENIZER, PROCESSOR
      MODEL_ID = model_id
      TOKENIZER = CLIPTokenizerFast.from_pretrained(MODEL_ID)
      MODEL = CLIPModel.from_pretrained(MODEL_ID)
      PROCESSOR = CLIPProcessor.from_pretrained(MODEL_ID)
  
  
  def embed_func(image):
      inputs = PROCESSOR(images=image, padded=True, return_tensors="pt")
      text_features = MODEL.get_image_features(**inputs)
      return text_features.detach().numpy()[0]
  
  
  from concurrent.futures import ThreadPoolExecutor
  
  db = lancedb.connect("data/video-lancedb")
  setup_clip_model("openai/clip-vit-base-patch32")
  
  
  def insert(video_ids, frames):
      with torch.no_grad():
          image_features = [
              embed_func(Image.open(f"./videos/{vid}/frame-{frame}.jpg"))
              for (vid, frame) in zip(video_ids, frames)
          ]
      if "videos" in db.table_names():
          table = db.open_table("videos")
          table.add(
              [
                  {
                      "vector": im,
                      "text": "",
                      "video_id": vid,
                      "start_time": (int(frame) - 1) * 30,
                  }
                  for (im, vid, frame) in zip(image_features, video_ids, frames)
              ]
          )
      else:
          db.create_table(
              "videos",
              [
                  {
                      "vector": im,
                      "text": "",
                      "video_id": vid,
                      "start_time": (int(frame) - 1) * 30,
                  }
                  for (im, vid, frame) in zip(image_features, video_ids, frames)
              ],
          )
  
  
  videos = [
      (
          re.search("(?<=videos\/).*(?=\/)", name).group(),
          re.search("(?<=frame-).*(?=.jpg)", name).group(),
      )
      for name in glob.glob("./videos/*/**")
  ]
  print(videos[:5])
  
  
  def process_video_chunk(chunk):
      video_ids, frames = zip(*chunk)
      insert(video_ids, frames)
  
  
  def threaded_video_processing(videos, chunk_size, max_workers):
      with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
          for i in range(0, len(videos), chunk_size):
              chunk = videos[i : i + chunk_size]
              executor.submit(process_video_chunk, chunk)
  
  
  # Assuming you have defined the insert function and videos list
  
  chunk_size = 500  # Number of videos to process in each chunk
  max_workers = 10  # Number of concurrent threads
  
  threaded_video_processing(videos, chunk_size, max_workers)
  
apis: lancedb.connect
extract_api: [((955, 992), 'lancedb.connect', 'lancedb.connect', (['"""data/video-lancedb"""'], {}), "('data/video-lancedb')\n", (970, 992), False, 'import lancedb\n'), ((553, 596), 'transformers.CLIPTokenizerFast.from_pretrained', 'CLIPTokenizerFast.from_pretrained', (['MODEL_ID'], {}), '(MODEL_ID)\n', (586, 596), False, 'from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n'), ((609, 644), 'transformers.CLIPModel.from_pretrained', 'CLIPModel.from_pretrained', (['MODEL_ID'], {}), '(MODEL_ID)\n', (634, 644), False, 'from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n'), ((661, 700), 'transformers.CLIPProcessor.from_pretrained', 'CLIPProcessor.from_pretrained', (['MODEL_ID'], {}), '(MODEL_ID)\n', (690, 700), False, 'from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n'), ((1084, 1099), 'torch.no_grad', 'torch.no_grad', ([], {}), '()\n', (1097, 1099), False, 'import torch\n'), ((2202, 2228), 'glob.glob', 'glob.glob', (['"""./videos/*/**"""'], {}), "('./videos/*/**')\n", (2211, 2228), False, 'import glob\n'), ((1151, 1198), 'PIL.Image.open', 'Image.open', (['f"""./videos/{vid}/frame-{frame}.jpg"""'], {}), "(f'./videos/{vid}/frame-{frame}.jpg')\n", (1161, 1198), False, 'from PIL import Image\n'), ((2072, 2114), 're.search', 're.search', (['"""(?<=videos\\\\/).*(?=\\\\/)"""', 'name'], {}), "('(?<=videos\\\\/).*(?=\\\\/)', name)\n", (2081, 2114), False, 'import re\n'), ((2130, 2170), 're.search', 're.search', (['"""(?<=frame-).*(?=.jpg)"""', 'name'], {}), "('(?<=frame-).*(?=.jpg)', name)\n", (2139, 2170), False, 'import re\n')]