code: import typer
  from lancedb import connect
  from rag_app.models import TextChunk, Document
  from pathlib import Path
  from typing import Iterable
  from tqdm import tqdm
  from rich import print
  import frontmatter
  import hashlib
  from datetime import datetime
  from unstructured.partition.text import partition_text
  
  app = typer.Typer()
  
  
  def read_files(path: Path, file_suffix: str) -> Iterable[Document]:
      for i, file in enumerate(path.iterdir()):
          if file.suffix != file_suffix:
              continue
          post = frontmatter.load(file)
          yield Document(
              id=hashlib.md5(post.content.encode("utf-8")).hexdigest(),
              content=post.content,
              filename=file.name,
              metadata=post.metadata,
          )
  
  
  def batch_chunks(chunks, batch_size=20):
      batch = []
      for chunk in chunks:
          batch.append(chunk)
          if len(batch) == batch_size:
              yield batch
              batch = []
  
      if batch:
          yield batch
  
  
  def chunk_text(
      documents: Iterable[Document], window_size: int = 1024, overlap: int = 0
  ):
      for doc in documents:
          for chunk_num, chunk in enumerate(partition_text(text=doc.content)):
              yield {
                  "doc_id": doc.id,
                  "chunk_id": chunk_num + 1,
                  "text": chunk.text,
                  "post_title": doc.metadata["title"],
                  "publish_date": datetime.strptime(doc.metadata["date"], "%Y-%m"),
                  "source": doc.metadata["url"],
              }
  
  
  @app.command(help="Ingest data into a given lancedb")
  def from_folder(
      db_path: str = typer.Option(help="Your LanceDB path"),
      table_name: str = typer.Option(help="Table to ingest data into"),
      folder_path: str = typer.Option(help="Folder to read data from"),
      file_suffix: str = typer.Option(default=".md", help="File suffix to filter by"),
  ):
      db = connect(db_path)
  
      if table_name not in db.table_names():
          db.create_table(table_name, schema=TextChunk, mode="overwrite")
  
      table = db.open_table(table_name)
      path = Path(folder_path)
  
      if not path.exists():
          raise ValueError(f"Ingestion folder of {folder_path} does not exist")
  
      files = read_files(path, file_suffix)
      chunks = chunk_text(files)
      batched_chunks = batch_chunks(chunks)
  
      ttl = 0
      for chunk_batch in tqdm(batched_chunks):
          table.add(chunk_batch)
          ttl += len(chunk_batch)
  
      print(f"Added {ttl} chunks to {table_name}")
  
apis: lancedb.connect
extract_api: [((312, 325), 'typer.Typer', 'typer.Typer', ([], {}), '()\n', (323, 325), False, 'import typer\n'), ((1598, 1636), 'typer.Option', 'typer.Option', ([], {'help': '"""Your LanceDB path"""'}), "(help='Your LanceDB path')\n", (1610, 1636), False, 'import typer\n'), ((1660, 1706), 'typer.Option', 'typer.Option', ([], {'help': '"""Table to ingest data into"""'}), "(help='Table to ingest data into')\n", (1672, 1706), False, 'import typer\n'), ((1731, 1776), 'typer.Option', 'typer.Option', ([], {'help': '"""Folder to read data from"""'}), "(help='Folder to read data from')\n", (1743, 1776), False, 'import typer\n'), ((1801, 1861), 'typer.Option', 'typer.Option', ([], {'default': '""".md"""', 'help': '"""File suffix to filter by"""'}), "(default='.md', help='File suffix to filter by')\n", (1813, 1861), False, 'import typer\n'), ((1875, 1891), 'lancedb.connect', 'connect', (['db_path'], {}), '(db_path)\n', (1882, 1891), False, 'from lancedb import connect\n'), ((2058, 2075), 'pathlib.Path', 'Path', (['folder_path'], {}), '(folder_path)\n', (2062, 2075), False, 'from pathlib import Path\n'), ((2333, 2353), 'tqdm.tqdm', 'tqdm', (['batched_chunks'], {}), '(batched_chunks)\n', (2337, 2353), False, 'from tqdm import tqdm\n'), ((2423, 2467), 'rich.print', 'print', (['f"""Added {ttl} chunks to {table_name}"""'], {}), "(f'Added {ttl} chunks to {table_name}')\n", (2428, 2467), False, 'from rich import print\n'), ((517, 539), 'frontmatter.load', 'frontmatter.load', (['file'], {}), '(file)\n', (533, 539), False, 'import frontmatter\n'), ((1142, 1174), 'unstructured.partition.text.partition_text', 'partition_text', ([], {'text': 'doc.content'}), '(text=doc.content)\n', (1156, 1174), False, 'from unstructured.partition.text import partition_text\n'), ((1395, 1443), 'datetime.datetime.strptime', 'datetime.strptime', (["doc.metadata['date']", '"""%Y-%m"""'], {}), "(doc.metadata['date'], '%Y-%m')\n", (1412, 1443), False, 'from datetime import datetime\n')]