code: import tempfile
  import time
  import os
  from utils import compute_sha1_from_file
  from langchain.schema import Document
  import streamlit as st
  from langchain.text_splitter import RecursiveCharacterTextSplitter
  from typing import List
  from sqlite3 import Connection
  from verse.sqlite_helper import *
  
  
  def update_metadata(conn: Connection, docs_with_metadata: List[Document]):
      insert_tuple = list(
          set(
              map(
                  lambda x: (
                      hash(x.metadata["file_sha1"]),
                      x.metadata["file_sha1"],
                      x.metadata["file_name"],
                  ),
                  docs_with_metadata,
              )
          )
      )
      insertmany(conn=conn, datalist=insert_tuple)
  
  
  def process_file(
      conn: Connection, file, loader_class, file_suffix, stats_db=None
  ) -> List[Document]:
      documents = []
      file_name = file.name
      file_size = file.size
      if st.secrets.self_hosted == "false":
          if file_size > 1000000:
              st.error(
                  "File size is too large. Please upload a file smaller than 1MB or self host."
              )
              return
  
      dateshort = time.strftime("%Y%m%d")
      with tempfile.NamedTemporaryFile(delete=False, suffix=file_suffix) as tmp_file:
          tmp_file.write(file.getvalue())
          tmp_file.flush()
  
          loader = loader_class(tmp_file.name)
          documents = loader.load()
          file_sha1 = compute_sha1_from_file(tmp_file.name)
  
      os.remove(tmp_file.name)
  
      chunk_size = st.session_state["chunk_size"]
      chunk_overlap = st.session_state["chunk_overlap"]
      print(f"Chunk Size {chunk_size} Overlap {chunk_overlap}")
  
      text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
          chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=["\n\n", ""]
      )
      documents = text_splitter.split_documents(documents)
  
      # Add the document sha1 as metadata to each document
      docs_with_metadata = [
          Document(
              page_content=doc.page_content,
              metadata={
                  "file_sha1": file_sha1,
                  "file_size": file_size,
                  "file_name": file_name,
                  "chunk_size": chunk_size,
                  "chunk_overlap": chunk_overlap,
                  "date": dateshort,
                  "file_type": file_suffix,
                  "page": doc.metadata["page"],
                  "dbsource": doc.metadata["source"]
              },
          )
          for doc in documents
      ]
  
      return docs_with_metadata
  
apis: langchain.schema.Document langchain.text_splitter.RecursiveCharacterTextSplitter.from_tiktoken_encoder
extract_api: [((1152, 1175), 'time.strftime', 'time.strftime', (['"""%Y%m%d"""'], {}), "('%Y%m%d')\n", (1165, 1175), False, 'import time\n'), ((1468, 1492), 'os.remove', 'os.remove', (['tmp_file.name'], {}), '(tmp_file.name)\n', (1477, 1492), False, 'import os\n'), ((1679, 1812), 'langchain.text_splitter.RecursiveCharacterTextSplitter.from_tiktoken_encoder', 'RecursiveCharacterTextSplitter.from_tiktoken_encoder', ([], {'chunk_size': 'chunk_size', 'chunk_overlap': 'chunk_overlap', 'separators': "['\\n\\n', '']"}), "(chunk_size=chunk_size,\n    chunk_overlap=chunk_overlap, separators=['\\n\\n', ''])\n", (1731, 1812), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1185, 1246), 'tempfile.NamedTemporaryFile', 'tempfile.NamedTemporaryFile', ([], {'delete': '(False)', 'suffix': 'file_suffix'}), '(delete=False, suffix=file_suffix)\n', (1212, 1246), False, 'import tempfile\n'), ((1425, 1462), 'utils.compute_sha1_from_file', 'compute_sha1_from_file', (['tmp_file.name'], {}), '(tmp_file.name)\n', (1447, 1462), False, 'from utils import compute_sha1_from_file\n'), ((1973, 2281), 'langchain.schema.Document', 'Document', ([], {'page_content': 'doc.page_content', 'metadata': "{'file_sha1': file_sha1, 'file_size': file_size, 'file_name': file_name,\n    'chunk_size': chunk_size, 'chunk_overlap': chunk_overlap, 'date':\n    dateshort, 'file_type': file_suffix, 'page': doc.metadata['page'],\n    'dbsource': doc.metadata['source']}"}), "(page_content=doc.page_content, metadata={'file_sha1': file_sha1,\n    'file_size': file_size, 'file_name': file_name, 'chunk_size':\n    chunk_size, 'chunk_overlap': chunk_overlap, 'date': dateshort,\n    'file_type': file_suffix, 'page': doc.metadata['page'], 'dbsource': doc\n    .metadata['source']})\n", (1981, 2281), False, 'from langchain.schema import Document\n'), ((998, 1095), 'streamlit.error', 'st.error', (['"""File size is too large. Please upload a file smaller than 1MB or self host."""'], {}), "(\n    'File size is too large. Please upload a file smaller than 1MB or self host.'\n    )\n", (1006, 1095), True, 'import streamlit as st\n')]