code: from dotenv import load_dotenv
  import os
  import lancedb
  import clip
  import torch
  from PIL import Image
  import glob
  import re
  
  from concurrent.futures import ThreadPoolExecutor
  import yt_dlp
  from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast
  
  # Set options for youtube_dl
  ydl_opts = {
      "retries": 0,
      "quiet": True,  # Silence youtube_dl output
      "extract_flat": True,  # Extract metadata only, no download
  }
  
  MODEL_ID = None
  MODEL = None
  TOKENIZER = None
  PROCESSOR = None
  
  
  def setup_clip_model(model_id):
      global MODEL_ID, MODEL, TOKENIZER, PROCESSOR
      MODEL_ID = model_id
      TOKENIZER = CLIPTokenizerFast.from_pretrained(MODEL_ID)
      MODEL = CLIPModel.from_pretrained(MODEL_ID)
      PROCESSOR = CLIPProcessor.from_pretrained(MODEL_ID)
  
  
  def embed_func(query):
      inputs = TOKENIZER([query], truncation=True, padding=True, return_tensors="pt")
      text_features = MODEL.get_text_features(**inputs)
      return text_features.detach().numpy()[0]
  
  
  def get_video_title(video_id):
      with yt_dlp.YoutubeDL(ydl_opts) as ydl:
          try:
              info = ydl.extract_info(
                  f"https://www.youtube.com/watch?v={video_id}", download=False
              )
              return info.get("title", None)
          except yt_dlp.utils.DownloadError:
              return None
  
  
  db = lancedb.connect("data/video-lancedb")
  
  setup_clip_model("openai/clip-vit-base-patch32")
  videos = list(
      set(
          [
              re.search("(?<=videos\/).*(?=\/)", name).group()
              for name in glob.glob("./videos/*/**")
          ]
      )
  )
  
  
  def insert(video_ids):
      titles = [(vid, get_video_title(vid)) for vid in video_ids]
      titles = [t for t in titles if t[1] is not None]
      video_ids, titles = zip(*titles)
      text_features = [embed_func(title) for title in titles]
      if "videos" in db.table_names():
          table = db.open_table("videos")
          table.add(
              [
                  {"vector": im, "text": title, "video_id": vid, "start_time": 0}
                  for (im, vid, title) in zip(text_features, video_ids, titles)
              ]
          )
      else:
          db.create_table(
              "videos",
              [
                  {"vector": im, "text": title, "video_id": vid, "start_time": 0}
                  for (im, vid, title) in zip(text_features, video_ids, titles)
              ],
          )
      print("done")
  
  
  import concurrent.futures
  
  
  def threaded_video_processing(videos, chunk_size, max_workers):
      with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
          for i in range(0, len(videos), chunk_size):
              chunk = videos[i : i + chunk_size]
              executor.submit(insert, chunk)
  
  
  # Assuming you have defined the insert function and videos list
  
  chunk_size = 500  # Number of videos to process in each chunk
  max_workers = 5  # Number of concurrent threads
  
  threaded_video_processing(videos, chunk_size, max_workers)
  
apis: lancedb.connect
extract_api: [((1313, 1350), 'lancedb.connect', 'lancedb.connect', (['"""data/video-lancedb"""'], {}), "('data/video-lancedb')\n", (1328, 1350), False, 'import lancedb\n'), ((621, 664), 'transformers.CLIPTokenizerFast.from_pretrained', 'CLIPTokenizerFast.from_pretrained', (['MODEL_ID'], {}), '(MODEL_ID)\n', (654, 664), False, 'from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n'), ((677, 712), 'transformers.CLIPModel.from_pretrained', 'CLIPModel.from_pretrained', (['MODEL_ID'], {}), '(MODEL_ID)\n', (702, 712), False, 'from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n'), ((729, 768), 'transformers.CLIPProcessor.from_pretrained', 'CLIPProcessor.from_pretrained', (['MODEL_ID'], {}), '(MODEL_ID)\n', (758, 768), False, 'from transformers import CLIPModel, CLIPProcessor, CLIPTokenizerFast\n'), ((1019, 1045), 'yt_dlp.YoutubeDL', 'yt_dlp.YoutubeDL', (['ydl_opts'], {}), '(ydl_opts)\n', (1035, 1045), False, 'import yt_dlp\n'), ((1520, 1546), 'glob.glob', 'glob.glob', (['"""./videos/*/**"""'], {}), "('./videos/*/**')\n", (1529, 1546), False, 'import glob\n'), ((1447, 1489), 're.search', 're.search', (['"""(?<=videos\\\\/).*(?=\\\\/)"""', 'name'], {}), "('(?<=videos\\\\/).*(?=\\\\/)', name)\n", (1456, 1489), False, 'import re\n')]