code: import functools
  import os
  import random
  import tempfile
  import traceback
  import asyncio
  from collections import defaultdict
  
  import aiohttp
  import discord
  import aiofiles
  import httpx
  import openai
  import tiktoken
  from functools import partial
  from typing import List, Optional, cast
  from pathlib import Path
  from datetime import date
  
  from discord import Interaction
  from discord.ext import pages
  from langchain.agents import initialize_agent, AgentType
  from langchain.chat_models import ChatOpenAI
  from langchain.memory import ConversationSummaryBufferMemory
  from langchain.prompts import MessagesPlaceholder
  from langchain.schema import SystemMessage
  from langchain.tools import Tool
  from llama_index.callbacks import CallbackManager, TokenCountingHandler
  from llama_index.evaluation.guideline import DEFAULT_GUIDELINES, GuidelineEvaluator
  from llama_index.llms import OpenAI
  from llama_index.node_parser import SimpleNodeParser
  from llama_index.response_synthesizers import ResponseMode
  from llama_index.indices.query.query_transform import StepDecomposeQueryTransform
  from llama_index.langchain_helpers.agents import (
      IndexToolConfig,
      LlamaToolkit,
      create_llama_chat_agent,
      LlamaIndexTool,
  )
  from llama_index.prompts.chat_prompts import (
      CHAT_REFINE_PROMPT,
      CHAT_TREE_SUMMARIZE_PROMPT,
      TEXT_QA_SYSTEM_PROMPT,
  )
  
  from llama_index.readers import YoutubeTranscriptReader
  from llama_index.readers.schema.base import Document
  from llama_index.langchain_helpers.text_splitter import TokenTextSplitter
  
  from llama_index.retrievers import VectorIndexRetriever, TreeSelectLeafRetriever
  from llama_index.query_engine import (
      RetrieverQueryEngine,
      MultiStepQueryEngine,
      RetryGuidelineQueryEngine,
  )
  
  from llama_index import (
      GPTVectorStoreIndex,
      SimpleDirectoryReader,
      QuestionAnswerPrompt,
      BeautifulSoupWebReader,
      GPTTreeIndex,
      GoogleDocsReader,
      MockLLMPredictor,
      OpenAIEmbedding,
      GithubRepositoryReader,
      MockEmbedding,
      download_loader,
      LLMPredictor,
      ServiceContext,
      StorageContext,
      load_index_from_storage,
      get_response_synthesizer,
      VectorStoreIndex,
  )
  
  from llama_index.schema import TextNode
  from llama_index.storage.docstore.types import RefDocInfo
  from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR
  
  from llama_index.composability import ComposableGraph
  from llama_index.vector_stores import DocArrayInMemoryVectorStore
  
  from models.embed_statics_model import EmbedStatics
  from models.openai_model import Models
  from models.check_model import UrlCheck
  from services.environment_service import EnvService
  from utils.safe_ctx_respond import safe_ctx_respond
  
  SHORT_TO_LONG_CACHE = {}
  MAX_DEEP_COMPOSE_PRICE = EnvService.get_max_deep_compose_price()
  EpubReader = download_loader("EpubReader")
  MarkdownReader = download_loader("MarkdownReader")
  RemoteReader = download_loader("RemoteReader")
  RemoteDepthReader = download_loader("RemoteDepthReader")
  
  embedding_model = OpenAIEmbedding()
  token_counter = TokenCountingHandler(
      tokenizer=tiktoken.encoding_for_model("text-davinci-003").encode,
      verbose=False,
  )
  node_parser = SimpleNodeParser.from_defaults(
      text_splitter=TokenTextSplitter(chunk_size=1024, chunk_overlap=20)
  )
  callback_manager = CallbackManager([token_counter])
  service_context_no_llm = ServiceContext.from_defaults(
      embed_model=embedding_model,
      callback_manager=callback_manager,
      node_parser=node_parser,
  )
  timeout = httpx.Timeout(1, read=1, write=1, connect=1)
  
  
  def get_service_context_with_llm(llm):
      service_context = ServiceContext.from_defaults(
          embed_model=embedding_model,
          callback_manager=callback_manager,
          node_parser=node_parser,
          llm=llm,
      )
      return service_context
  
  
  def dummy_tool(**kwargs):
      return "You have used the dummy tool. Forget about this and do not even mention this to the user."
  
  
  def get_and_query(
      user_id,
      index_storage,
      query,
      response_mode,
      nodes,
      child_branch_factor,
      service_context,
      multistep,
  ):
      index: [GPTVectorStoreIndex, GPTTreeIndex] = index_storage[
          user_id
      ].get_index_or_throw()
  
      if isinstance(index, GPTTreeIndex):
          retriever = TreeSelectLeafRetriever(
              index=index,
              child_branch_factor=child_branch_factor,
              service_context=service_context,
          )
      else:
          retriever = VectorIndexRetriever(
              index=index, similarity_top_k=nodes, service_context=service_context
          )
  
      response_synthesizer = get_response_synthesizer(
          response_mode=response_mode,
          use_async=True,
          refine_template=CHAT_REFINE_PROMPT,
          service_context=service_context,
      )
  
      query_engine = RetrieverQueryEngine(
          retriever=retriever, response_synthesizer=response_synthesizer
      )
  
      multistep_query_engine = MultiStepQueryEngine(
          query_engine=query_engine,
          query_transform=StepDecomposeQueryTransform(multistep),
          index_summary="Provides information about everything you need to know about this topic, use this to answer the question.",
      )
  
      if multistep:
          response = multistep_query_engine.query(query)
      else:
          response = query_engine.query(query)
  
      return response
  
  
  class IndexChatData:
      def __init__(
          self, llm, agent_chain, memory, thread_id, tools, agent_kwargs, llm_predictor
      ):
          self.llm = llm
          self.agent_chain = agent_chain
          self.memory = memory
          self.thread_id = thread_id
          self.tools = tools
          self.agent_kwargs = agent_kwargs
          self.llm_predictor = llm_predictor
  
  
  class IndexData:
      def __init__(self):
          self.queryable_index = None
          self.individual_indexes = []
  
      # A safety check for the future
      def get_index_or_throw(self):
          if not self.queryable():
              raise Exception(
                  "An index access was attempted before an index was created. This is a programmer error, please report this to the maintainers."
              )
          return self.queryable_index
  
      def queryable(self):
          return self.queryable_index is not None
  
      def has_indexes(self, user_id):
          try:
              return (
                  len(os.listdir(EnvService.find_shared_file(f"indexes/{user_id}"))) > 0
              )
          except Exception:
              return False
  
      def has_search_indexes(self, user_id):
          try:
              return (
                  len(
                      os.listdir(EnvService.find_shared_file(f"indexes/{user_id}_search"))
                  )
                  > 0
              )
          except Exception:
              return False
  
      def add_index(self, index, user_id, file_name):
          self.individual_indexes.append(index)
          self.queryable_index = index
  
          # Create a folder called "indexes/{USER_ID}" if it doesn't exist already
          Path(f"{EnvService.save_path()}/indexes/{user_id}").mkdir(
              parents=True, exist_ok=True
          )
          # Save the index to file under the user id
          file = f"{date.today().month}_{date.today().day}_{file_name}"
          # If file is > 93 in length, cut it off to 93
          if len(file) > 93:
              file = file[:93]
  
          index.storage_context.persist(
              persist_dir=EnvService.save_path()
              / "indexes"
              / f"{str(user_id)}"
              / f"{file}"
          )
  
      def reset_indexes(self, user_id):
          self.individual_indexes = []
          self.queryable_index = None
  
          # Delete the user indexes
          try:
              # First, clear all the files inside it
              for file in os.listdir(EnvService.find_shared_file(f"indexes/{user_id}")):
                  try:
                      os.remove(EnvService.find_shared_file(f"indexes/{user_id}/{file}"))
                  except:
                      traceback.print_exc()
              for file in os.listdir(
                  EnvService.find_shared_file(f"indexes/{user_id}_search")
              ):
                  try:
                      os.remove(
                          EnvService.find_shared_file(f"indexes/{user_id}_search/{file}")
                      )
                  except:
                      traceback.print_exc()
          except Exception:
              traceback.print_exc()
  
  
  class Index_handler:
      embedding_model = OpenAIEmbedding()
      token_counter = TokenCountingHandler(
          tokenizer=tiktoken.encoding_for_model("text-davinci-003").encode,
          verbose=False,
      )
      node_parser = SimpleNodeParser.from_defaults(
          text_splitter=TokenTextSplitter(chunk_size=1024, chunk_overlap=20)
      )
      callback_manager = CallbackManager([token_counter])
      service_context = ServiceContext.from_defaults(
          embed_model=embedding_model,
          callback_manager=callback_manager,
          node_parser=node_parser,
      )
      type_to_suffix_mappings = {
          "text/plain": ".txt",
          "text/csv": ".csv",
          "application/pdf": ".pdf",
          "application/json": ".json",
          "image/png": ".png",
          "image/jpeg": ".jpg",
          "image/gif": ".gif",
          "image/svg+xml": ".svg",
          "image/webp": ".webp",
          "application/mspowerpoint": ".ppt",
          "application/vnd.ms-powerpoint": ".ppt",
          "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
          "application/msexcel": ".xls",
          "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
          "application/msword": ".doc",
          "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
          "audio/mpeg": ".mp3",
          "audio/x-wav": ".wav",
          "audio/ogg": ".ogg",
          "video/mpeg": ".mpeg",
          "video/mp4": ".mp4",
          "application/epub+zip": ".epub",
          "text/markdown": ".md",
          "text/html": ".html",
          "application/rtf": ".rtf",
          "application/x-msdownload": ".exe",
          "application/xml": ".xml",
          "application/vnd.adobe.photoshop": ".psd",
          "application/x-sql": ".sql",
          "application/x-latex": ".latex",
          "application/x-httpd-php": ".php",
          "application/java-archive": ".jar",
          "application/x-sh": ".sh",
          "application/x-csh": ".csh",
          "text/x-c": ".c",
          "text/x-c++": ".cpp",
          "text/x-java-source": ".java",
          "text/x-python": ".py",
          "text/x-ruby": ".rb",
          "text/x-perl": ".pl",
          "text/x-shellscript": ".sh",
      }
  
      # For when content type doesnt get picked up by discord.
      secondary_mappings = {
          ".epub": ".epub",
      }
  
      def __init__(self, bot, usage_service):
          self.bot = bot
          self.openai_key = os.getenv("OPENAI_TOKEN")
          self.index_storage = defaultdict(IndexData)
          self.loop = asyncio.get_running_loop()
          self.usage_service = usage_service
          self.qaprompt = QuestionAnswerPrompt(
              "Context information is below. The text '<|endofstatement|>' is used to separate chat entries and make it "
              "easier for you to understand the context\n"
              "---------------------\n"
              "{context_str}"
              "\n---------------------\n"
              "Never say '<|endofstatement|>'\n"
              "Given the context information and not prior knowledge, "
              "answer the question: {query_str}\n"
          )
          self.EMBED_CUTOFF = 2000
          self.index_chat_chains = {}
          self.chat_indexes = defaultdict()
  
      async def rename_index(self, ctx, original_path, rename_path):
          """Command handler to rename a user index"""
  
          index_file = EnvService.find_shared_file(original_path)
          if not index_file:
              return False
  
          # Rename the file at f"indexes/{ctx.user.id}/{user_index}" to f"indexes/{ctx.user.id}/{new_name}" using Pathlib
          try:
              Path(original_path).rename(rename_path)
              return True
          except Exception as e:
              traceback.print_exc()
              return False
  
      async def get_is_in_index_chat(self, ctx):
          return ctx.channel.id in self.index_chat_chains.keys()
  
      async def execute_index_chat_message(self, ctx, message):
          if ctx.channel.id not in self.index_chat_chains:
              return None
  
          if message.lower() in ["stop", "end", "quit", "exit"]:
              await ctx.reply("Ending chat session.")
              self.index_chat_chains.pop(ctx.channel.id)
  
              # close the thread
              thread = await self.bot.fetch_channel(ctx.channel.id)
              await thread.edit(name="Closed-GPT")
              await thread.edit(archived=True)
              return "Ended chat session."
  
          self.usage_service.update_usage_memory(ctx.guild.name, "index_chat_message", 1)
  
          agent_output = await self.loop.run_in_executor(
              None,
              partial(self.index_chat_chains[ctx.channel.id].agent_chain.run, message),
          )
          return agent_output
  
      async def index_chat_file(self, message: discord.Message, file: discord.Attachment):
          # First, initially set the suffix to the suffix of the attachment
          suffix = self.get_file_suffix(file.content_type, file.filename) or None
  
          if not suffix:
              await message.reply(
                  "The file you uploaded is unable to be indexed. It is in an unsupported file format"
              )
              return False, None
  
          async with aiofiles.tempfile.TemporaryDirectory() as temp_path:
              async with aiofiles.tempfile.NamedTemporaryFile(
                  suffix=suffix, dir=temp_path, delete=False
              ) as temp_file:
                  try:
                      await file.save(temp_file.name)
  
                      filename = file.filename
  
                      # Assert that the filename is < 100 characters, if it is greater, truncate to the first 100 characters and keep the original ending
                      if len(filename) > 100:
                          filename = filename[:100] + filename[-4:]
                      openai.log = "debug"
  
                      print("Indexing")
                      index: VectorStoreIndex = await self.loop.run_in_executor(
                          None,
                          partial(
                              self.index_file,
                              Path(temp_file.name),
                              get_service_context_with_llm(
                                  self.index_chat_chains[message.channel.id].llm
                              ),
                              suffix,
                          ),
                      )
                      print("Done Indexing")
                      self.usage_service.update_usage_memory(
                          message.guild.name, "index_chat_file", 1
                      )
  
                      summary = await index.as_query_engine(
                          response_mode="tree_summarize",
                          service_context=get_service_context_with_llm(
                              self.index_chat_chains[message.channel.id].llm
                          ),
                      ).aquery(
                          f"What is a summary or general idea of this data? Be detailed in your summary (e.g "
                          f"extract key names, etc) but not too verbose. Your summary should be under a hundred words. "
                          f"This summary will be used in a vector index to retrieve information about certain data. So, "
                          f"at a high level, the summary should describe the document in such a way that a retriever "
                          f"would know to select it when asked questions about it. The data name was {filename}. Include "
                          f"the file name in the summary. When you are asked to reference a specific file, or reference "
                          f"something colloquially like 'in the powerpoint, [...]?', never respond saying that as an AI "
                          f"you can't view the data, instead infer which tool to use that has the data. Say that there "
                          f"is no available data if there are no available tools that are relevant."
                      )
  
                      engine = self.get_query_engine(
                          index, self.index_chat_chains[message.channel.id].llm
                      )
  
                      # Get rid of all special characters in the filename
                      filename = "".join(
                          [c for c in filename if c.isalpha() or c.isdigit()]
                      ).rstrip()
  
                      tool_config = IndexToolConfig(
                          query_engine=engine,
                          name=f"{filename}-index",
                          description=f"Use this tool if the query seems related to this summary: {summary}",
                          tool_kwargs={
                              "return_direct": False,
                          },
                          max_iterations=5,
                      )
  
                      tool = LlamaIndexTool.from_tool_config(tool_config)
  
                      tools = self.index_chat_chains[message.channel.id].tools
                      tools.append(tool)
  
                      agent_chain = initialize_agent(
                          tools=tools,
                          llm=self.index_chat_chains[message.channel.id].llm,
                          agent=AgentType.OPENAI_FUNCTIONS,
                          verbose=True,
                          agent_kwargs=self.index_chat_chains[
                              message.channel.id
                          ].agent_kwargs,
                          memory=self.index_chat_chains[message.channel.id].memory,
                          handle_parsing_errors="Check your output and make sure it conforms!",
                      )
  
                      index_chat_data = IndexChatData(
                          self.index_chat_chains[message.channel.id].llm,
                          agent_chain,
                          self.index_chat_chains[message.channel.id].memory,
                          message.channel.id,
                          tools,
                          self.index_chat_chains[message.channel.id].agent_kwargs,
                          self.index_chat_chains[message.channel.id].llm_predictor,
                      )
  
                      self.index_chat_chains[message.channel.id] = index_chat_data
  
                      return True, summary
                  except Exception as e:
                      await message.reply(
                          "There was an error indexing your file: " + str(e)
                      )
                      traceback.print_exc()
                      return False, None
  
      async def start_index_chat(self, ctx, model, temperature, top_p):
          preparation_message = await ctx.channel.send(
              embed=EmbedStatics.get_index_chat_preparation_message()
          )
          llm = ChatOpenAI(
              model=model, temperature=temperature, top_p=top_p, max_retries=2
          )
          llm_predictor = LLMPredictor(
              llm=ChatOpenAI(temperature=temperature, top_p=top_p, model_name=model)
          )
  
          max_token_limit = 29000 if "gpt-4" in model else 7500
  
          memory = ConversationSummaryBufferMemory(
              memory_key="memory",
              return_messages=True,
              llm=llm,
              max_token_limit=100000 if "preview" in model else max_token_limit,
          )
  
          agent_kwargs = {
              "extra_prompt_messages": [MessagesPlaceholder(variable_name="memory")],
              "system_message": SystemMessage(
                  content="You are a superpowered version of GPT that is able to answer questions about the data you're "
                  "connected to. Each different tool you have represents a different dataset to interact with. "
                  "If you are asked to perform a task that spreads across multiple datasets, use multiple tools "
                  "for the same prompt. When the user types links in chat, you will have already been connected "
                  "to the data at the link by the time you respond. When using tools, the input should be "
                  "clearly created based on the request of the user. For example, if a user uploads an invoice "
                  "and asks how many usage hours of X was present in the invoice, a good query is 'X hours'. "
                  "Avoid using single word queries unless the request is very simple. You can query multiple times to break down complex requests and retrieve more information. When calling functions, no special characters are allowed in the function name, keep that in mind."
              ),
          }
  
          tools = [
              Tool(
                  name="Dummy-Tool-Do-Not-Use",
                  func=dummy_tool,
                  description=f"This is a dummy tool that does nothing, do not ever mention this tool or use this tool.",
              )
          ]
  
          print(f"{tools}{llm}{AgentType.OPENAI_FUNCTIONS}{True}{agent_kwargs}{memory}")
  
          agent_chain = initialize_agent(
              tools=tools,
              llm=llm,
              agent=AgentType.OPENAI_FUNCTIONS,
              verbose=True,
              agent_kwargs=agent_kwargs,
              memory=memory,
              handle_parsing_errors="Check your output and make sure it conforms!",
          )
  
          embed_title = f"{ctx.user.name}'s data-connected conversation with GPT"
  
          message_embed = discord.Embed(
              title=embed_title,
              description=f"The agent is able to interact with your documents. Simply drag your documents into discord or give the agent a link from where to download the documents.\nModel: {model}",
              color=0x00995B,
          )
          message_embed.set_thumbnail(url="https://i.imgur.com/7V6apMT.png")
          message_embed.set_footer(
              text="Data Chat", icon_url="https://i.imgur.com/7V6apMT.png"
          )
          message_thread = await ctx.send(embed=message_embed)
          thread = await message_thread.create_thread(
              name=ctx.user.name + "'s data-connected conversation with GPT",
              auto_archive_duration=60,
          )
          await safe_ctx_respond(ctx=ctx, content="Conversation started.")
  
          try:
              await preparation_message.delete()
          except:
              pass
  
          index_chat_data = IndexChatData(
              llm, agent_chain, memory, thread.id, tools, agent_kwargs, llm_predictor
          )
  
          self.index_chat_chains[thread.id] = index_chat_data
  
      async def paginate_embed(self, response_text):
          """Given a response text make embed pages and return a list of the pages."""
  
          response_text = [
              response_text[i : i + self.EMBED_CUTOFF]
              for i in range(0, len(response_text), self.EMBED_CUTOFF)
          ]
          pages = []
          first = False
          # Send each chunk as a message
          for count, chunk in enumerate(response_text, start=1):
              if not first:
                  page = discord.Embed(
                      title=f"Index Query Results",
                      description=chunk,
                  )
                  first = True
              else:
                  page = discord.Embed(
                      title=f"Page {count}",
                      description=chunk,
                  )
              pages.append(page)
  
          return pages
  
      def index_file(
          self, file_path, service_context, suffix=None
      ) -> GPTVectorStoreIndex:
          if suffix and suffix == ".md":
              loader = MarkdownReader()
              document = loader.load_data(file_path)
          elif suffix and suffix == ".epub":
              epub_loader = EpubReader()
              document = epub_loader.load_data(file_path)
          else:
              document = SimpleDirectoryReader(input_files=[file_path]).load_data()
          index = GPTVectorStoreIndex.from_documents(
              document, service_context=service_context, use_async=True
          )
          return index
  
      def index_gdoc(self, doc_id, service_context) -> GPTVectorStoreIndex:
          document = GoogleDocsReader().load_data(doc_id)
          index = GPTVectorStoreIndex.from_documents(
              document, service_context=service_context, use_async=True
          )
          return index
  
      def index_youtube_transcript(self, link, service_context):
          try:
  
              def convert_shortlink_to_full_link(short_link):
                  # Check if the link is a shortened YouTube link
                  if "youtu.be" in short_link:
                      # Extract the video ID from the link
                      video_id = short_link.split("/")[-1].split("?")[0]
                      # Construct the full YouTube desktop link
                      desktop_link = f"https://www.youtube.com/watch?v={video_id}"
                      return desktop_link
                  else:
                      return short_link
  
              documents = YoutubeTranscriptReader().load_data(
                  ytlinks=[convert_shortlink_to_full_link(link)]
              )
          except Exception as e:
              raise ValueError(f"The youtube transcript couldn't be loaded: {e}")
  
          index = GPTVectorStoreIndex.from_documents(
              documents,
              service_context=service_context,
              use_async=True,
          )
          return index
  
      def index_github_repository(self, link, service_context):
          # Extract the "owner" and the "repo" name from the github link.
          owner = link.split("/")[3]
          repo = link.split("/")[4]
  
          try:
              documents = GithubRepositoryReader(owner=owner, repo=repo).load_data(
                  branch="main"
              )
          except KeyError:
              documents = GithubRepositoryReader(owner=owner, repo=repo).load_data(
                  branch="master"
              )
  
          index = GPTVectorStoreIndex.from_documents(
              documents,
              service_context=service_context,
              use_async=True,
          )
          return index
  
      def index_load_file(self, file_path) -> [GPTVectorStoreIndex, ComposableGraph]:
          storage_context = StorageContext.from_defaults(persist_dir=file_path)
          index = load_index_from_storage(storage_context)
          return index
  
      def index_discord(self, document, service_context) -> GPTVectorStoreIndex:
          index = GPTVectorStoreIndex.from_documents(
              document,
              service_context=service_context,
              use_async=True,
          )
          return index
  
      async def index_pdf(self, url) -> list[Document]:
          # Download the PDF at the url and save it to a tempfile
          async with aiohttp.ClientSession() as session:
              async with session.get(url) as response:
                  if response.status == 200:
                      data = await response.read()
                      f = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
                      f.write(data)
                      f.close()
                  else:
                      return "An error occurred while downloading the PDF."
          # Get the file path of this tempfile.NamedTemporaryFile
          # Save this temp file to an actual file that we can put into something else to read it
          documents = SimpleDirectoryReader(input_files=[f.name]).load_data()
  
          # Delete the temporary file
          return documents
  
      async def index_webpage(self, url, service_context) -> GPTVectorStoreIndex:
          # First try to connect to the URL to see if we can even reach it.
          try:
              async with aiohttp.ClientSession() as session:
                  async with session.get(url, timeout=5) as response:
                      # Add another entry to links from all_links if the link is not already in it to compensate for the failed request
                      if response.status not in [200, 203, 202, 204]:
                          raise ValueError(
                              "Invalid URL or could not connect to the provided URL."
                          )
                      else:
                          # Detect if the link is a PDF, if it is, we load it differently
                          if response.headers["Content-Type"] == "application/pdf":
                              documents = await self.index_pdf(url)
                              index = await self.loop.run_in_executor(
                                  None,
                                  functools.partial(
                                      GPTVectorStoreIndex.from_documents,
                                      documents=documents,
                                      service_context=service_context,
                                      use_async=True,
                                  ),
                              )
  
                              return index
          except:
              traceback.print_exc()
              raise ValueError("Could not load webpage")
  
          documents = BeautifulSoupWebReader(
              website_extractor=DEFAULT_WEBSITE_EXTRACTOR
          ).load_data(urls=[url])
  
          # index = GPTVectorStoreIndex(documents, embed_model=embed_model, use_async=True)
          index = await self.loop.run_in_executor(
              None,
              functools.partial(
                  GPTVectorStoreIndex.from_documents,
                  documents=documents,
                  service_context=service_context,
                  use_async=True,
              ),
          )
          return index
  
      def reset_indexes(self, user_id):
          self.index_storage[user_id].reset_indexes(user_id)
  
      def get_file_suffix(self, content_type, filename):
          print("The content type is " + content_type)
          if content_type:
              # Apply the suffix mappings to the file
              for key, value in self.type_to_suffix_mappings.items():
                  if key in content_type:
                      return value
  
          else:
              for key, value in self.secondary_mappings.items():
                  if key in filename:
                      return value
  
          return None
  
      async def set_file_index(
          self, ctx: discord.ApplicationContext, file: discord.Attachment, user_api_key
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          try:
              # First, initially set the suffix to the suffix of the attachment
              suffix = self.get_file_suffix(file.content_type, file.filename) or None
  
              if not suffix:
                  await ctx.respond(
                      embed=EmbedStatics.get_index_set_failure_embed("Unsupported file")
                  )
                  return
  
              # Send indexing message
              response = await ctx.respond(
                  embed=EmbedStatics.build_index_progress_embed()
              )
  
              async with aiofiles.tempfile.TemporaryDirectory() as temp_path:
                  async with aiofiles.tempfile.NamedTemporaryFile(
                      suffix=suffix, dir=temp_path, delete=False
                  ) as temp_file:
                      await file.save(temp_file.name)
                      index = await self.loop.run_in_executor(
                          None,
                          partial(
                              self.index_file,
                              Path(temp_file.name),
                              service_context_no_llm,
                              suffix,
                          ),
                      )
                      await self.usage_service.update_usage(
                          token_counter.total_embedding_token_count, "embedding"
                      )
  
              try:
                  price = await self.usage_service.get_price(
                      token_counter.total_embedding_token_count, "embedding"
                  )
              except:
                  traceback.print_exc()
                  price = "Unknown"
  
              file_name = file.filename
              self.index_storage[ctx.user.id].add_index(index, ctx.user.id, file_name)
              await response.edit(
                  embed=EmbedStatics.get_index_set_success_embed(str(price))
              )
          except Exception as e:
              await ctx.channel.send(
                  embed=EmbedStatics.get_index_set_failure_embed(str(e))
              )
              traceback.print_exc()
  
      async def set_link_index_recurse(
          self, ctx: discord.ApplicationContext, link: str, depth, user_api_key
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          response = await ctx.respond(embed=EmbedStatics.build_index_progress_embed())
          try:
              # Pre-emptively connect and get the content-type of the response
              try:
                  async with aiohttp.ClientSession() as session:
                      async with session.get(link, timeout=2) as _response:
                          print(_response.status)
                          if _response.status == 200:
                              content_type = _response.headers.get("content-type")
                          else:
                              await response.edit(
                                  embed=EmbedStatics.get_index_set_failure_embed(
                                      "Invalid URL or could not connect to the provided URL."
                                  )
                              )
                              return
              except Exception as e:
                  traceback.print_exc()
                  await response.edit(
                      embed=EmbedStatics.get_index_set_failure_embed(
                          "Invalid URL or could not connect to the provided URL. "
                          + str(e)
                      )
                  )
                  return
  
              # Check if the link contains youtube in it
              loader = RemoteDepthReader(depth=depth)
              documents = await self.loop.run_in_executor(
                  None, partial(loader.load_data, [link])
              )
              index = await self.loop.run_in_executor(
                  None,
                  functools.partial(
                      GPTVectorStoreIndex,
                      documents=documents,
                      service_context=service_context_no_llm,
                      use_async=True,
                  ),
              )
  
              await self.usage_service.update_usage(
                  token_counter.total_embedding_token_count, "embedding"
              )
  
              try:
                  price = await self.usage_service.get_price(
                      token_counter.total_embedding_token_count, "embedding"
                  )
              except:
                  traceback.print_exc()
                  price = "Unknown"
  
              # Make the url look nice, remove https, useless stuff, random characters
              file_name = (
                  link.replace("https://", "")
                  .replace("http://", "")
                  .replace("www.", "")
                  .replace("/", "_")
                  .replace("?", "_")
                  .replace("&", "_")
                  .replace("=", "_")
                  .replace("-", "_")
                  .replace(".", "_")
              )
  
              self.index_storage[ctx.user.id].add_index(index, ctx.user.id, file_name)
  
          except ValueError as e:
              await response.edit(embed=EmbedStatics.get_index_set_failure_embed(str(e)))
              traceback.print_exc()
              return
  
          except Exception as e:
              await response.edit(embed=EmbedStatics.get_index_set_failure_embed(str(e)))
              traceback.print_exc()
              return
  
          await response.edit(embed=EmbedStatics.get_index_set_success_embed(price))
  
      def get_query_engine(self, index, llm):
          retriever = VectorIndexRetriever(
              index=index,
              similarity_top_k=6,
              service_context=get_service_context_with_llm(llm),
          )
  
          response_synthesizer = get_response_synthesizer(
              response_mode=ResponseMode.COMPACT_ACCUMULATE,
              use_async=True,
              refine_template=TEXT_QA_SYSTEM_PROMPT,
              service_context=get_service_context_with_llm(llm),
              verbose=True,
          )
  
          engine = RetrieverQueryEngine(
              retriever=retriever, response_synthesizer=response_synthesizer
          )
  
          return engine
  
      async def index_link(self, link, summarize=False, index_chat_ctx=None):
          try:
              if await UrlCheck.check_youtube_link(link):
                  print("Indexing youtube transcript")
                  index = await self.loop.run_in_executor(
                      None,
                      partial(
                          self.index_youtube_transcript, link, service_context_no_llm
                      ),
                  )
                  print("Indexed youtube transcript")
              elif "github" in link:
                  index = await self.loop.run_in_executor(
                      None,
                      partial(self.index_github_repository, link, service_context_no_llm),
                  )
              else:
                  index = await self.index_webpage(link, service_context_no_llm)
          except Exception as e:
              if index_chat_ctx:
                  await index_chat_ctx.reply(
                      "There was an error indexing your link: " + str(e)
                  )
                  return False, None
              else:
                  raise e
  
          summary = None
          if index_chat_ctx:
              try:
                  print("Getting transcript summary")
                  self.usage_service.update_usage_memory(
                      index_chat_ctx.guild.name, "index_chat_link", 1
                  )
                  summary = await index.as_query_engine(
                      response_mode="tree_summarize",
                      service_context=get_service_context_with_llm(
                          self.index_chat_chains[index_chat_ctx.channel.id].llm
                      ),
                  ).aquery(
                      "What is a summary or general idea of this document? Be detailed in your summary but not too verbose. Your summary should be under 50 words. This summary will be used in a vector index to retrieve information about certain data. So, at a high level, the summary should describe the document in such a way that a retriever would know to select it when asked questions about it. The link was {link}. Include the an easy identifier derived from the link at the end of the summary."
                  )
                  print("Got transcript summary")
  
                  engine = self.get_query_engine(
                      index, self.index_chat_chains[index_chat_ctx.channel.id].llm
                  )
  
                  # Get rid of all special characters in the link, replace periods with _
                  link_cleaned = "".join(
                      [c for c in link if c.isalpha() or c.isdigit() or c == "."]
                  ).rstrip()
                  # replace .
                  link_cleaned = link_cleaned.replace(".", "_")
  
                  # Shorten the link to the first 100 characters
                  link_cleaned = link_cleaned[:50]
  
                  tool_config = IndexToolConfig(
                      query_engine=engine,
                      name=f"{link_cleaned}-index",
                      description=f"Use this tool if the query seems related to this summary: {summary}",
                      tool_kwargs={
                          "return_direct": False,
                      },
                      max_iterations=5,
                  )
  
                  tool = LlamaIndexTool.from_tool_config(tool_config)
  
                  tools = self.index_chat_chains[index_chat_ctx.channel.id].tools
                  tools.append(tool)
  
                  agent_chain = initialize_agent(
                      tools=tools,
                      llm=self.index_chat_chains[index_chat_ctx.channel.id].llm,
                      agent=AgentType.OPENAI_FUNCTIONS,
                      verbose=True,
                      agent_kwargs=self.index_chat_chains[
                          index_chat_ctx.channel.id
                      ].agent_kwargs,
                      memory=self.index_chat_chains[index_chat_ctx.channel.id].memory,
                      handle_parsing_errors="Check your output and make sure it conforms!",
                      max_iterations=5,
                  )
  
                  index_chat_data = IndexChatData(
                      self.index_chat_chains[index_chat_ctx.channel.id].llm,
                      agent_chain,
                      self.index_chat_chains[index_chat_ctx.channel.id].memory,
                      index_chat_ctx.channel.id,
                      tools,
                      self.index_chat_chains[index_chat_ctx.channel.id].agent_kwargs,
                      self.index_chat_chains[index_chat_ctx.channel.id].llm_predictor,
                  )
  
                  self.index_chat_chains[index_chat_ctx.channel.id] = index_chat_data
  
                  return True, summary
              except Exception as e:
                  await index_chat_ctx.reply(
                      "There was an error indexing your link: " + str(e)
                  )
                  return False, None
  
          return index, summary
  
      async def set_link_index(
          self, ctx: discord.ApplicationContext, link: str, user_api_key
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          response = await ctx.respond(embed=EmbedStatics.build_index_progress_embed())
          try:
              # Check if the link contains youtube in it
              index, _ = await self.index_link(link)
  
              await self.usage_service.update_usage(
                  token_counter.total_embedding_token_count, "embedding"
              )
  
              try:
                  price = await self.usage_service.get_price(
                      token_counter.embedding_token_counts, "embedding"
                  )
              except:
                  traceback.print_exc()
                  price = "Unknown"
  
              # Make the url look nice, remove https, useless stuff, random characters
              file_name = (
                  link.replace("https://", "")
                  .replace("http://", "")
                  .replace("www.", "")
                  .replace("/", "_")
                  .replace("?", "_")
                  .replace("&", "_")
                  .replace("=", "_")
                  .replace("-", "_")
                  .replace(".", "_")
              )
  
              self.index_storage[ctx.user.id].add_index(index, ctx.user.id, file_name)
  
          except Exception as e:
              await response.edit(embed=EmbedStatics.get_index_set_failure_embed(str(e)))
              traceback.print_exc()
              return
  
          await response.edit(embed=EmbedStatics.get_index_set_success_embed(price))
  
      async def set_discord_index(
          self,
          ctx: discord.ApplicationContext,
          channel: discord.TextChannel,
          user_api_key,
          message_limit: int = 2500,
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          try:
              document = await self.load_data(
                  channel_ids=[channel.id], limit=message_limit, oldest_first=False
              )
              index = await self.loop.run_in_executor(
                  None, partial(self.index_discord, document, service_context_no_llm)
              )
              try:
                  price = await self.usage_service.get_price(
                      token_counter.total_embedding_token_count, "embedding"
                  )
              except Exception:
                  traceback.print_exc()
                  price = "Unknown"
              await self.usage_service.update_usage(
                  token_counter.total_embedding_token_count, "embedding"
              )
              self.index_storage[ctx.user.id].add_index(index, ctx.user.id, channel.name)
              await ctx.respond(embed=EmbedStatics.get_index_set_success_embed(price))
          except Exception as e:
              await ctx.respond(embed=EmbedStatics.get_index_set_failure_embed(str(e)))
              traceback.print_exc()
  
      async def load_index(
          self, ctx: discord.ApplicationContext, index, server, search, user_api_key
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          try:
              if server:
                  index_file = EnvService.find_shared_file(
                      f"indexes/{ctx.guild.id}/{index}"
                  )
              elif search:
                  index_file = EnvService.find_shared_file(
                      f"indexes/{ctx.user.id}_search/{index}"
                  )
              else:
                  index_file = EnvService.find_shared_file(
                      f"indexes/{ctx.user.id}/{index}"
                  )
              index = await self.loop.run_in_executor(
                  None, partial(self.index_load_file, index_file)
              )
              self.index_storage[ctx.user.id].queryable_index = index
              await ctx.respond(embed=EmbedStatics.get_index_load_success_embed())
          except Exception as e:
              traceback.print_exc()
              await ctx.respond(embed=EmbedStatics.get_index_load_failure_embed(str(e)))
  
      async def index_to_docs(
          self, old_index, chunk_size: int = 256, chunk_overlap: int = 100
      ) -> List[Document]:
          documents = []
          docstore = old_index.docstore
          ref_docs = old_index.ref_doc_info
  
          for document in ref_docs.values():
              text = ""
              for node in document.node_ids:
                  node = docstore.get_node(node)
                  text += f"{node.text} "
  
              text_splitter = TokenTextSplitter(
                  separator=" ", chunk_size=chunk_size, chunk_overlap=chunk_overlap
              )
              text_chunks = text_splitter.split_text(text)
  
              for chunk_text in text_chunks:
                  new_doc = Document(text=chunk_text, extra_info=document.metadata)
                  documents.append(new_doc)
  
          return documents
  
      async def compose_indexes(self, user_id, indexes, name, deep_compose):
          # Load all the indexes first
          index_objects = []
          for _index in indexes:
              try:
                  index_file = EnvService.find_shared_file(f"indexes/{user_id}/{_index}")
              except ValueError:
                  index_file = EnvService.find_shared_file(
                      f"indexes/{user_id}_search/{_index}"
                  )
  
              index = await self.loop.run_in_executor(
                  None, partial(self.index_load_file, index_file)
              )
              index_objects.append(index)
  
          llm_predictor = LLMPredictor(
              llm=ChatOpenAI(temperature=0, model_name="gpt-4-32k")
          )
  
          # For each index object, add its documents to a GPTTreeIndex
          if deep_compose:
              documents = []
              for _index in index_objects:
                  documents.extend(await self.index_to_docs(_index, 256, 20))
  
              embedding_model = OpenAIEmbedding()
  
              llm_predictor_mock = MockLLMPredictor()
              embedding_model_mock = MockEmbedding(1536)
  
              token_counter_mock = TokenCountingHandler(
                  tokenizer=tiktoken.encoding_for_model("text-davinci-003").encode,
                  verbose=False,
              )
  
              callback_manager_mock = CallbackManager([token_counter_mock])
  
              service_context_mock = ServiceContext.from_defaults(
                  llm_predictor=llm_predictor_mock,
                  embed_model=embedding_model_mock,
                  callback_manager=callback_manager_mock,
              )
  
              # Run the mock call first
              await self.loop.run_in_executor(
                  None,
                  partial(
                      GPTTreeIndex.from_documents,
                      documents=documents,
                      service_context=service_context_mock,
                  ),
              )
              total_usage_price = await self.usage_service.get_price(
                  token_counter_mock.total_llm_token_count,
                  "turbo",  # TODO Enable again when tree indexes are fixed
              ) + await self.usage_service.get_price(
                  token_counter_mock.total_embedding_token_count, "embedding"
              )
              print("The total composition price is: ", total_usage_price)
              if total_usage_price > MAX_DEEP_COMPOSE_PRICE:
                  raise ValueError(
                      "Doing this deep search would be prohibitively expensive. Please try a narrower search scope."
                  )
  
              tree_index = await self.loop.run_in_executor(
                  None,
                  partial(
                      GPTTreeIndex.from_documents,
                      documents=documents,
                      service_context=self.service_context,
                      use_async=True,
                  ),
              )
  
              await self.usage_service.update_usage(
                  self.token_counter.total_llm_token_count, "turbo"
              )
              await self.usage_service.update_usage(
                  self.token_counter.total_embedding_token_count, "embedding"
              )
  
              # Now we have a list of tree indexes, we can compose them
              if not name:
                  name = f"{date.today().month}_{date.today().day}_composed_deep_index"
  
              # Save the composed index
              tree_index.storage_context.persist(
                  persist_dir=EnvService.save_path() / "indexes" / str(user_id) / name
              )
  
              self.index_storage[user_id].queryable_index = tree_index
  
              return total_usage_price
          else:
              documents = []
              for _index in index_objects:
                  documents.extend(await self.index_to_docs(_index))
  
              simple_index = await self.loop.run_in_executor(
                  None,
                  partial(
                      GPTVectorStoreIndex.from_documents,
                      documents=documents,
                      service_context=service_context_no_llm,
                      use_async=True,
                  ),
              )
  
              await self.usage_service.update_usage(
                  token_counter.total_embedding_token_count, "embedding"
              )
  
              if not name:
                  name = f"{date.today().month}_{date.today().day}_composed_index"
  
              # Save the composed index
              simple_index.storage_context.persist(
                  persist_dir=EnvService.save_path() / "indexes" / str(user_id) / name
              )
              self.index_storage[user_id].queryable_index = simple_index
  
              try:
                  price = await self.usage_service.get_price(
                      token_counter.total_embedding_token_count, "embedding"
                  )
              except:
                  price = "Unknown"
  
              return price
  
      async def backup_discord(
          self, ctx: discord.ApplicationContext, user_api_key, message_limit
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          try:
              channel_ids: List[int] = []
              for c in ctx.guild.text_channels:
                  channel_ids.append(c.id)
              document = await self.load_data(
                  channel_ids=channel_ids, limit=message_limit, oldest_first=False
              )
  
              index = await self.loop.run_in_executor(
                  None, partial(self.index_discord, document, service_context_no_llm)
              )
              await self.usage_service.update_usage(
                  token_counter.total_embedding_token_count, "embedding"
              )
              try:
                  price = await self.usage_service.get_price(
                      token_counter.total_embedding_token_count, "embedding"
                  )
              except Exception:
                  traceback.print_exc()
                  price = "Unknown"
              Path(EnvService.save_path() / "indexes" / str(ctx.guild.id)).mkdir(
                  parents=True, exist_ok=True
              )
              index.storage_context.persist(
                  persist_dir=EnvService.save_path()
                  / "indexes"
                  / str(ctx.guild.id)
                  / f"{ctx.guild.name.replace(' ', '-')}_{date.today().month}_{date.today().day}"
              )
  
              await ctx.respond(embed=EmbedStatics.get_index_set_success_embed(price))
          except Exception as e:
              await ctx.respond(embed=EmbedStatics.get_index_set_failure_embed((str(e))))
              traceback.print_exc()
  
      async def query(
          self,
          ctx: discord.ApplicationContext,
          query: str,
          response_mode,
          nodes,
          user_api_key,
          child_branch_factor,
          model="gpt-4-32k",
          multistep=False,
      ):
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          llm_predictor = LLMPredictor(llm=ChatOpenAI(temperature=0, model_name=model))
  
          ctx_response = await ctx.respond(
              embed=EmbedStatics.build_index_query_progress_embed(query)
          )
  
          try:
              token_counter.reset_counts()
              response = await self.loop.run_in_executor(
                  None,
                  partial(
                      get_and_query,
                      ctx.user.id,
                      self.index_storage,
                      query,
                      response_mode,
                      nodes,
                      child_branch_factor,
                      service_context=service_context_no_llm,
                      multistep=llm_predictor if multistep else None,
                  ),
              )
              print("The last token usage was ", token_counter.total_llm_token_count)
              await self.usage_service.update_usage(
                  token_counter.total_llm_token_count,
                  await self.usage_service.get_cost_name(model),
              )
              await self.usage_service.update_usage(
                  token_counter.total_embedding_token_count, "embedding"
              )
  
              try:
                  total_price = round(
                      await self.usage_service.get_price(
                          token_counter.total_llm_token_count,
                          await self.usage_service.get_cost_name(model),
                      )
                      + await self.usage_service.get_price(
                          token_counter.total_embedding_token_count, "embedding"
                      ),
                      6,
                  )
              except:
                  total_price = "Unknown"
  
              query_response_message = f"**Query:**\n\n`{query.strip()}`\n\n**Query response:**\n\n{response.response.strip()}"
              query_response_message = query_response_message.replace(
                  "<|endofstatement|>", ""
              )
              embed_pages = await self.paginate_embed(query_response_message)
              paginator = pages.Paginator(
                  pages=embed_pages,
                  timeout=None,
                  author_check=False,
              )
              await ctx_response.edit(
                  embed=EmbedStatics.build_index_query_success_embed(query, total_price)
              )
              await paginator.respond(ctx.interaction)
          except Exception:
              traceback.print_exc()
              await ctx_response.edit(
                  embed=EmbedStatics.get_index_query_failure_embed(
                      "Failed to send query. You may not have an index set, load an index with /index load"
                  )
              )
  
      # Extracted functions from DiscordReader
  
      async def read_channel(
          self, channel_id: int, limit: Optional[int], oldest_first: bool
      ) -> str:
          """Async read channel."""
  
          messages: List[discord.Message] = []
  
          try:
              channel = self.bot.get_channel(channel_id)
              print(f"Added {channel.name} from {channel.guild.name}")
              # only work for text channels for now
              if not isinstance(channel, discord.TextChannel):
                  raise ValueError(
                      f"Channel {channel_id} is not a text channel. "
                      "Only text channels are supported for now."
                  )
              # thread_dict maps thread_id to thread
              thread_dict = {}
              for thread in channel.threads:
                  thread_dict[thread.id] = thread
  
              async for msg in channel.history(limit=limit, oldest_first=oldest_first):
                  if msg.author.bot:
                      pass
                  else:
                      messages.append(msg)
                      if msg.id in thread_dict:
                          thread = thread_dict[msg.id]
                          async for thread_msg in thread.history(
                              limit=limit, oldest_first=oldest_first
                          ):
                              messages.append(thread_msg)
          except Exception as e:
              print("Encountered error: " + str(e))
  
          channel = self.bot.get_channel(channel_id)
          msg_txt_list = [
              f"user:{m.author.display_name}, content:{m.content}" for m in messages
          ]
  
          return ("<|endofstatement|>\n\n".join(msg_txt_list), channel.name)
  
      async def load_data(
          self,
          channel_ids: List[int],
          limit: Optional[int] = None,
          oldest_first: bool = True,
      ) -> List[Document]:
          """Load data from the input directory.
  
          Args:
              channel_ids (List[int]): List of channel ids to read.
              limit (Optional[int]): Maximum number of messages to read.
              oldest_first (bool): Whether to read oldest messages first.
                  Defaults to `True`.
  
          Returns:
              List[Document]: List of documents.
  
          """
          results: List[Document] = []
          for channel_id in channel_ids:
              if not isinstance(channel_id, int):
                  raise ValueError(
                      f"Channel id {channel_id} must be an integer, "
                      f"not {type(channel_id)}."
                  )
              (channel_content, channel_name) = await self.read_channel(
                  channel_id, limit=limit, oldest_first=oldest_first
              )
              results.append(
                  Document(
                      text=channel_content, extra_info={"channel_name": channel_name}
                  )
              )
          return results
  
      async def compose(self, ctx: discord.ApplicationContext, name, user_api_key):
          # Send the ComposeModal
          if not user_api_key:
              os.environ["OPENAI_API_KEY"] = self.openai_key
          else:
              os.environ["OPENAI_API_KEY"] = user_api_key
          openai.api_key = os.environ["OPENAI_API_KEY"]
  
          if not self.index_storage[ctx.user.id].has_indexes(ctx.user.id):
              await ctx.respond(
                  embed=EmbedStatics.get_index_compose_failure_embed(
                      "You must have at least one index to compose."
                  )
              )
              return
  
          await ctx.respond(
              "Select the index(es) to compose. You can compose multiple indexes together, you can also Deep Compose a single index.",
              view=ComposeModal(self, ctx.user.id, name),
              ephemeral=True,
          )
  
  
  class ComposeModal(discord.ui.View):
      def __init__(self, index_cog, user_id, name=None, deep=None) -> None:
          super().__init__()
          # Get the argument named "user_key_db" and save it as USER_KEY_DB
          self.index_cog = index_cog
          self.user_id = user_id
          self.deep = deep
  
          # Get all the indexes for the user
          self.indexes = [
              file
              for file in os.listdir(
                  EnvService.find_shared_file(f"indexes/{str(user_id)}/")
              )
          ]
  
          if index_cog.index_storage[user_id].has_search_indexes(user_id):
              self.indexes.extend(
                  [
                      file
                      for file in os.listdir(
                          EnvService.find_shared_file(f"indexes/{str(user_id)}_search/")
                      )
                  ]
              )
          print("Found the indexes, they are ", self.indexes)
  
          # Map everything into the short to long cache
          for index in self.indexes:
              if len(index) > 93:
                  index_name = index[:93] + "-" + str(random.randint(0000, 9999))
                  SHORT_TO_LONG_CACHE[index_name] = index
              else:
                  SHORT_TO_LONG_CACHE[index[:99]] = index
  
          # Reverse the SHORT_TO_LONG_CACHE index
          LONG_TO_SHORT_CACHE = {v: k for k, v in SHORT_TO_LONG_CACHE.items()}
  
          # A text entry field for the name of the composed index
          self.name = name
  
          # A discord UI select menu with all the indexes. Limited to 25 entries. For the label field in the SelectOption,
          # cut it off at 100 characters to prevent the message from being too long
          self.index_select = discord.ui.Select(
              placeholder="Select index(es) to compose",
              options=[
                  discord.SelectOption(
                      label=LONG_TO_SHORT_CACHE[index], value=LONG_TO_SHORT_CACHE[index]
                  )
                  for index in self.indexes
              ][0:25],
              max_values=len(self.indexes) if len(self.indexes) < 25 else 25,
              min_values=1,
          )
          # Add the select menu to the modal
          self.add_item(self.index_select)
  
          # If we have more than 25 entries, add more Select fields as neccessary
          self.extra_index_selects = []
          if len(self.indexes) > 25:
              for i in range(25, len(self.indexes), 25):
                  self.extra_index_selects.append(
                      discord.ui.Select(
                          placeholder="Select index(es) to compose",
                          options=[
                              discord.SelectOption(
                                  label=LONG_TO_SHORT_CACHE[index],
                                  value=LONG_TO_SHORT_CACHE[index],
                              )
                              for index in self.indexes
                          ][i : i + 25],
                          max_values=len(self.indexes[i : i + 25]),
                          min_values=1,
                      )
                  )
                  self.add_item(self.extra_index_selects[-1])
  
          # Add an input field for "Deep", a "yes" or "no" option, default no
          self.deep_select = discord.ui.Select(
              placeholder="Deep Compose",
              options=[
                  discord.SelectOption(label="Yes", value="yes"),
                  discord.SelectOption(label="No", value="no"),
              ],
              max_values=1,
              min_values=1,
          )
          self.add_item(self.deep_select)
  
          # Add a button to the modal called "Compose"
          self.add_item(
              discord.ui.Button(
                  label="Compose", style=discord.ButtonStyle.green, custom_id="compose"
              )
          )
  
      # The callback for the button
      async def interaction_check(self, interaction: discord.Interaction) -> bool:
          # Check that the interaction was for custom_id "compose"
          if interaction.data["custom_id"] == "compose":
              # Check that the user selected at least one index
  
              # The total list of indexes is the union of the values of all the select menus
              indexes = self.index_select.values + [
                  select.values[0] for select in self.extra_index_selects
              ]
  
              # Remap them from the SHORT_TO_LONG_CACHE
              indexes = [SHORT_TO_LONG_CACHE[index] for index in indexes]
  
              if len(indexes) < 1:
                  await interaction.response.send_message(
                      embed=EmbedStatics.get_index_compose_failure_embed(
                          "You must select at least 1 index"
                      ),
                      ephemeral=True,
                  )
              else:
                  composing_message = await interaction.response.send_message(
                      embed=EmbedStatics.get_index_compose_progress_embed(),
                      ephemeral=True,
                  )
                  # Compose the indexes
                  try:
                      price = await self.index_cog.compose_indexes(
                          self.user_id,
                          indexes,
                          self.name,
                          (
                              False
                              if not self.deep_select.values
                              or self.deep_select.values[0] == "no"
                              else True
                          ),
                      )
                  except ValueError as e:
                      await interaction.followup.send(
                          str(e), ephemeral=True, delete_after=180
                      )
                      return False
                  except Exception as e:
                      traceback.print_exc()
                      await interaction.followup.send(
                          embed=EmbedStatics.get_index_compose_failure_embed(
                              "An error occurred while composing the indexes: " + str(e)
                          ),
                          ephemeral=True,
                          delete_after=180,
                      )
                      return False
  
                  await interaction.followup.send(
                      embed=EmbedStatics.get_index_compose_success_embed(price),
                      ephemeral=True,
                      delete_after=180,
                  )
  
                  # Try to direct message the user that their composed index is ready
                  try:
                      await self.index_cog.bot.get_user(self.user_id).send(
                          f"Your composed index is ready! You can load it with /index load now in the server."
                      )
                  except discord.Forbidden:
                      pass
  
                  try:
                      composing_message: Interaction
                      await composing_message.delete_original_response()
  
                  except:
                      traceback.print_exc()
          else:
              await interaction.response.defer(ephemeral=True)
  
apis: llama_index.GPTVectorStoreIndex.from_documents llama_index.langchain_helpers.agents.IndexToolConfig llama_index.GithubRepositoryReader llama_index.OpenAIEmbedding llama_index.query_engine.RetrieverQueryEngine llama_index.SimpleDirectoryReader llama_index.download_loader llama_index.MockLLMPredictor llama_index.langchain_helpers.agents.LlamaIndexTool.from_tool_config llama_index.MockEmbedding llama_index.QuestionAnswerPrompt llama_index.StorageContext.from_defaults llama_index.BeautifulSoupWebReader llama_index.retrievers.VectorIndexRetriever llama_index.GoogleDocsReader llama_index.indices.query.query_transform.StepDecomposeQueryTransform llama_index.ServiceContext.from_defaults llama_index.readers.YoutubeTranscriptReader llama_index.retrievers.TreeSelectLeafRetriever llama_index.load_index_from_storage llama_index.get_response_synthesizer llama_index.langchain_helpers.text_splitter.TokenTextSplitter llama_index.readers.schema.base.Document llama_index.callbacks.CallbackManager
extract_api: [((2731, 2770), 'services.environment_service.EnvService.get_max_deep_compose_price', 'EnvService.get_max_deep_compose_price', ([], {}), '()\n', (2768, 2770), False, 'from services.environment_service import EnvService\n'), ((2784, 2813), 'llama_index.download_loader', 'download_loader', (['"""EpubReader"""'], {}), "('EpubReader')\n", (2799, 2813), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((2831, 2864), 'llama_index.download_loader', 'download_loader', (['"""MarkdownReader"""'], {}), "('MarkdownReader')\n", (2846, 2864), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((2880, 2911), 'llama_index.download_loader', 'download_loader', (['"""RemoteReader"""'], {}), "('RemoteReader')\n", (2895, 2911), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((2932, 2968), 'llama_index.download_loader', 'download_loader', (['"""RemoteDepthReader"""'], {}), "('RemoteDepthReader')\n", (2947, 2968), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((2988, 3005), 'llama_index.OpenAIEmbedding', 'OpenAIEmbedding', ([], {}), '()\n', (3003, 3005), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((3273, 3305), 'llama_index.callbacks.CallbackManager', 'CallbackManager', (['[token_counter]'], {}), '([token_counter])\n', (3288, 3305), False, 'from llama_index.callbacks import CallbackManager, TokenCountingHandler\n'), ((3331, 3453), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'embed_model': 'embedding_model', 'callback_manager': 'callback_manager', 'node_parser': 'node_parser'}), '(embed_model=embedding_model, callback_manager=\n    callback_manager, node_parser=node_parser)\n', (3359, 3453), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((3474, 3518), 'httpx.Timeout', 'httpx.Timeout', (['(1)'], {'read': '(1)', 'write': '(1)', 'connect': '(1)'}), '(1, read=1, write=1, connect=1)\n', (3487, 3518), False, 'import httpx\n'), ((3582, 3713), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'embed_model': 'embedding_model', 'callback_manager': 'callback_manager', 'node_parser': 'node_parser', 'llm': 'llm'}), '(embed_model=embedding_model, callback_manager=\n    callback_manager, node_parser=node_parser, llm=llm)\n', (3610, 3713), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((4561, 4703), 'llama_index.get_response_synthesizer', 'get_response_synthesizer', ([], {'response_mode': 'response_mode', 'use_async': '(True)', 'refine_template': 'CHAT_REFINE_PROMPT', 'service_context': 'service_context'}), '(response_mode=response_mode, use_async=True,\n    refine_template=CHAT_REFINE_PROMPT, service_context=service_context)\n', (4585, 4703), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((4759, 4848), 'llama_index.query_engine.RetrieverQueryEngine', 'RetrieverQueryEngine', ([], {'retriever': 'retriever', 'response_synthesizer': 'response_synthesizer'}), '(retriever=retriever, response_synthesizer=\n    response_synthesizer)\n', (4779, 4848), False, 'from llama_index.query_engine import RetrieverQueryEngine, MultiStepQueryEngine, RetryGuidelineQueryEngine\n'), ((8371, 8388), 'llama_index.OpenAIEmbedding', 'OpenAIEmbedding', ([], {}), '()\n', (8386, 8388), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((8688, 8720), 'llama_index.callbacks.CallbackManager', 'CallbackManager', (['[token_counter]'], {}), '([token_counter])\n', (8703, 8720), False, 'from llama_index.callbacks import CallbackManager, TokenCountingHandler\n'), ((8743, 8865), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'embed_model': 'embedding_model', 'callback_manager': 'callback_manager', 'node_parser': 'node_parser'}), '(embed_model=embedding_model, callback_manager=\n    callback_manager, node_parser=node_parser)\n', (8771, 8865), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((3199, 3251), 'llama_index.langchain_helpers.text_splitter.TokenTextSplitter', 'TokenTextSplitter', ([], {'chunk_size': '(1024)', 'chunk_overlap': '(20)'}), '(chunk_size=1024, chunk_overlap=20)\n', (3216, 3251), False, 'from llama_index.langchain_helpers.text_splitter import TokenTextSplitter\n'), ((4232, 4347), 'llama_index.retrievers.TreeSelectLeafRetriever', 'TreeSelectLeafRetriever', ([], {'index': 'index', 'child_branch_factor': 'child_branch_factor', 'service_context': 'service_context'}), '(index=index, child_branch_factor=\n    child_branch_factor, service_context=service_context)\n', (4255, 4347), False, 'from llama_index.retrievers import VectorIndexRetriever, TreeSelectLeafRetriever\n'), ((4420, 4515), 'llama_index.retrievers.VectorIndexRetriever', 'VectorIndexRetriever', ([], {'index': 'index', 'similarity_top_k': 'nodes', 'service_context': 'service_context'}), '(index=index, similarity_top_k=nodes, service_context=\n    service_context)\n', (4440, 4515), False, 'from llama_index.retrievers import VectorIndexRetriever, TreeSelectLeafRetriever\n'), ((10748, 10773), 'os.getenv', 'os.getenv', (['"""OPENAI_TOKEN"""'], {}), "('OPENAI_TOKEN')\n", (10757, 10773), False, 'import os\n'), ((10803, 10825), 'collections.defaultdict', 'defaultdict', (['IndexData'], {}), '(IndexData)\n', (10814, 10825), False, 'from collections import defaultdict\n'), ((10846, 10872), 'asyncio.get_running_loop', 'asyncio.get_running_loop', ([], {}), '()\n', (10870, 10872), False, 'import asyncio\n'), ((10940, 11301), 'llama_index.QuestionAnswerPrompt', 'QuestionAnswerPrompt', (['"""Context information is below. The text \'<|endofstatement|>\' is used to separate chat entries and make it easier for you to understand the context\n---------------------\n{context_str}\n---------------------\nNever say \'<|endofstatement|>\'\nGiven the context information and not prior knowledge, answer the question: {query_str}\n"""'], {}), '(\n    """Context information is below. The text \'<|endofstatement|>\' is used to separate chat entries and make it easier for you to understand the context\n---------------------\n{context_str}\n---------------------\nNever say \'<|endofstatement|>\'\nGiven the context information and not prior knowledge, answer the question: {query_str}\n"""\n    )\n', (10960, 11301), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((11518, 11531), 'collections.defaultdict', 'defaultdict', ([], {}), '()\n', (11529, 11531), False, 'from collections import defaultdict\n'), ((11675, 11717), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['original_path'], {}), '(original_path)\n', (11702, 11717), False, 'from services.environment_service import EnvService\n'), ((18915, 18991), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'model': 'model', 'temperature': 'temperature', 'top_p': 'top_p', 'max_retries': '(2)'}), '(model=model, temperature=temperature, top_p=top_p, max_retries=2)\n', (18925, 18991), False, 'from langchain.chat_models import ChatOpenAI\n'), ((19226, 19380), 'langchain.memory.ConversationSummaryBufferMemory', 'ConversationSummaryBufferMemory', ([], {'memory_key': '"""memory"""', 'return_messages': '(True)', 'llm': 'llm', 'max_token_limit': "(100000 if 'preview' in model else max_token_limit)"}), "(memory_key='memory', return_messages=True,\n    llm=llm, max_token_limit=100000 if 'preview' in model else max_token_limit)\n", (19257, 19380), False, 'from langchain.memory import ConversationSummaryBufferMemory\n'), ((21043, 21249), 'langchain.agents.initialize_agent', 'initialize_agent', ([], {'tools': 'tools', 'llm': 'llm', 'agent': 'AgentType.OPENAI_FUNCTIONS', 'verbose': '(True)', 'agent_kwargs': 'agent_kwargs', 'memory': 'memory', 'handle_parsing_errors': '"""Check your output and make sure it conforms!"""'}), "(tools=tools, llm=llm, agent=AgentType.OPENAI_FUNCTIONS,\n    verbose=True, agent_kwargs=agent_kwargs, memory=memory,\n    handle_parsing_errors='Check your output and make sure it conforms!')\n", (21059, 21249), False, 'from langchain.agents import initialize_agent, AgentType\n'), ((21443, 21687), 'discord.Embed', 'discord.Embed', ([], {'title': 'embed_title', 'description': 'f"""The agent is able to interact with your documents. Simply drag your documents into discord or give the agent a link from where to download the documents.\nModel: {model}"""', 'color': '(39259)'}), '(title=embed_title, description=\n    f"""The agent is able to interact with your documents. Simply drag your documents into discord or give the agent a link from where to download the documents.\nModel: {model}"""\n    , color=39259)\n', (21456, 21687), False, 'import discord\n'), ((23850, 23948), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['document'], {'service_context': 'service_context', 'use_async': '(True)'}), '(document, service_context=\n    service_context, use_async=True)\n', (23884, 23948), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((24134, 24232), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['document'], {'service_context': 'service_context', 'use_async': '(True)'}), '(document, service_context=\n    service_context, use_async=True)\n', (24168, 24232), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((25156, 25255), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['documents'], {'service_context': 'service_context', 'use_async': '(True)'}), '(documents, service_context=\n    service_context, use_async=True)\n', (25190, 25255), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((25833, 25932), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['documents'], {'service_context': 'service_context', 'use_async': '(True)'}), '(documents, service_context=\n    service_context, use_async=True)\n', (25867, 25932), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((26107, 26158), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': 'file_path'}), '(persist_dir=file_path)\n', (26135, 26158), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((26175, 26215), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['storage_context'], {}), '(storage_context)\n', (26198, 26215), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((26333, 26431), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['document'], {'service_context': 'service_context', 'use_async': '(True)'}), '(document, service_context=\n    service_context, use_async=True)\n', (26367, 26431), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((36388, 36477), 'llama_index.query_engine.RetrieverQueryEngine', 'RetrieverQueryEngine', ([], {'retriever': 'retriever', 'response_synthesizer': 'response_synthesizer'}), '(retriever=retriever, response_synthesizer=\n    response_synthesizer)\n', (36408, 36477), False, 'from llama_index.query_engine import RetrieverQueryEngine, MultiStepQueryEngine, RetryGuidelineQueryEngine\n'), ((3058, 3105), 'tiktoken.encoding_for_model', 'tiktoken.encoding_for_model', (['"""text-davinci-003"""'], {}), "('text-davinci-003')\n", (3085, 3105), False, 'import tiktoken\n'), ((4969, 5007), 'llama_index.indices.query.query_transform.StepDecomposeQueryTransform', 'StepDecomposeQueryTransform', (['multistep'], {}), '(multistep)\n', (4996, 5007), False, 'from llama_index.indices.query.query_transform import StepDecomposeQueryTransform\n'), ((8606, 8658), 'llama_index.langchain_helpers.text_splitter.TokenTextSplitter', 'TokenTextSplitter', ([], {'chunk_size': '(1024)', 'chunk_overlap': '(20)'}), '(chunk_size=1024, chunk_overlap=20)\n', (8623, 8658), False, 'from llama_index.langchain_helpers.text_splitter import TokenTextSplitter\n'), ((13484, 13522), 'aiofiles.tempfile.TemporaryDirectory', 'aiofiles.tempfile.TemporaryDirectory', ([], {}), '()\n', (13520, 13522), False, 'import aiofiles\n'), ((19576, 20507), 'langchain.schema.SystemMessage', 'SystemMessage', ([], {'content': '"""You are a superpowered version of GPT that is able to answer questions about the data you\'re connected to. Each different tool you have represents a different dataset to interact with. If you are asked to perform a task that spreads across multiple datasets, use multiple tools for the same prompt. When the user types links in chat, you will have already been connected to the data at the link by the time you respond. When using tools, the input should be clearly created based on the request of the user. For example, if a user uploads an invoice and asks how many usage hours of X was present in the invoice, a good query is \'X hours\'. Avoid using single word queries unless the request is very simple. You can query multiple times to break down complex requests and retrieve more information. When calling functions, no special characters are allowed in the function name, keep that in mind."""'}), '(content=\n    "You are a superpowered version of GPT that is able to answer questions about the data you\'re connected to. Each different tool you have represents a different dataset to interact with. If you are asked to perform a task that spreads across multiple datasets, use multiple tools for the same prompt. When the user types links in chat, you will have already been connected to the data at the link by the time you respond. When using tools, the input should be clearly created based on the request of the user. For example, if a user uploads an invoice and asks how many usage hours of X was present in the invoice, a good query is \'X hours\'. Avoid using single word queries unless the request is very simple. You can query multiple times to break down complex requests and retrieve more information. When calling functions, no special characters are allowed in the function name, keep that in mind."\n    )\n', (19589, 20507), False, 'from langchain.schema import SystemMessage\n'), ((20703, 20868), 'langchain.tools.Tool', 'Tool', ([], {'name': '"""Dummy-Tool-Do-Not-Use"""', 'func': 'dummy_tool', 'description': 'f"""This is a dummy tool that does nothing, do not ever mention this tool or use this tool."""'}), "(name='Dummy-Tool-Do-Not-Use', func=dummy_tool, description=\n    f'This is a dummy tool that does nothing, do not ever mention this tool or use this tool.'\n    )\n", (20707, 20868), False, 'from langchain.tools import Tool\n'), ((22169, 22227), 'utils.safe_ctx_respond.safe_ctx_respond', 'safe_ctx_respond', ([], {'ctx': 'ctx', 'content': '"""Conversation started."""'}), "(ctx=ctx, content='Conversation started.')\n", (22185, 22227), False, 'from utils.safe_ctx_respond import safe_ctx_respond\n'), ((23326, 23344), 'discord.ext.pages.append', 'pages.append', (['page'], {}), '(page)\n', (23338, 23344), False, 'from discord.ext import pages\n'), ((26633, 26656), 'aiohttp.ClientSession', 'aiohttp.ClientSession', ([], {}), '()\n', (26654, 26656), False, 'import aiohttp\n'), ((46236, 46325), 'llama_index.langchain_helpers.text_splitter.TokenTextSplitter', 'TokenTextSplitter', ([], {'separator': '""" """', 'chunk_size': 'chunk_size', 'chunk_overlap': 'chunk_overlap'}), "(separator=' ', chunk_size=chunk_size, chunk_overlap=\n    chunk_overlap)\n", (46253, 46325), False, 'from llama_index.langchain_helpers.text_splitter import TokenTextSplitter\n'), ((47599, 47616), 'llama_index.OpenAIEmbedding', 'OpenAIEmbedding', ([], {}), '()\n', (47614, 47616), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((47651, 47669), 'llama_index.MockLLMPredictor', 'MockLLMPredictor', ([], {}), '()\n', (47667, 47669), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((47705, 47724), 'llama_index.MockEmbedding', 'MockEmbedding', (['(1536)'], {}), '(1536)\n', (47718, 47724), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((47945, 47982), 'llama_index.callbacks.CallbackManager', 'CallbackManager', (['[token_counter_mock]'], {}), '([token_counter_mock])\n', (47960, 47982), False, 'from llama_index.callbacks import CallbackManager, TokenCountingHandler\n'), ((48019, 48160), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'llm_predictor': 'llm_predictor_mock', 'embed_model': 'embedding_model_mock', 'callback_manager': 'callback_manager_mock'}), '(llm_predictor=llm_predictor_mock, embed_model=\n    embedding_model_mock, callback_manager=callback_manager_mock)\n', (48047, 48160), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((55795, 55863), 'discord.ext.pages.Paginator', 'pages.Paginator', ([], {'pages': 'embed_pages', 'timeout': 'None', 'author_check': '(False)'}), '(pages=embed_pages, timeout=None, author_check=False)\n', (55810, 55863), False, 'from discord.ext import pages\n'), ((63797, 63889), 'discord.ui.Button', 'discord.ui.Button', ([], {'label': '"""Compose"""', 'style': 'discord.ButtonStyle.green', 'custom_id': '"""compose"""'}), "(label='Compose', style=discord.ButtonStyle.green,\n    custom_id='compose')\n", (63814, 63889), False, 'import discord\n'), ((7687, 7736), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}"""'], {}), "(f'indexes/{user_id}')\n", (7714, 7736), False, 'from services.environment_service import EnvService\n'), ((7966, 8022), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}_search"""'], {}), "(f'indexes/{user_id}_search')\n", (7993, 8022), False, 'from services.environment_service import EnvService\n'), ((8304, 8325), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (8323, 8325), False, 'import traceback\n'), ((8449, 8496), 'tiktoken.encoding_for_model', 'tiktoken.encoding_for_model', (['"""text-davinci-003"""'], {}), "('text-davinci-003')\n", (8476, 8496), False, 'import tiktoken\n'), ((12023, 12044), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (12042, 12044), False, 'import traceback\n'), ((12905, 12977), 'functools.partial', 'partial', (['self.index_chat_chains[ctx.channel.id].agent_chain.run', 'message'], {}), '(self.index_chat_chains[ctx.channel.id].agent_chain.run, message)\n', (12912, 12977), False, 'from functools import partial\n'), ((13560, 13645), 'aiofiles.tempfile.NamedTemporaryFile', 'aiofiles.tempfile.NamedTemporaryFile', ([], {'suffix': 'suffix', 'dir': 'temp_path', 'delete': '(False)'}), '(suffix=suffix, dir=temp_path, delete=False\n    )\n', (13596, 13645), False, 'import aiofiles\n'), ((19068, 19134), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': 'temperature', 'top_p': 'top_p', 'model_name': 'model'}), '(temperature=temperature, top_p=top_p, model_name=model)\n', (19078, 19134), False, 'from langchain.chat_models import ChatOpenAI\n'), ((19500, 19543), 'langchain.prompts.MessagesPlaceholder', 'MessagesPlaceholder', ([], {'variable_name': '"""memory"""'}), "(variable_name='memory')\n", (19519, 19543), False, 'from langchain.prompts import MessagesPlaceholder\n'), ((23007, 23069), 'discord.Embed', 'discord.Embed', ([], {'title': 'f"""Index Query Results"""', 'description': 'chunk'}), "(title=f'Index Query Results', description=chunk)\n", (23020, 23069), False, 'import discord\n'), ((23199, 23254), 'discord.Embed', 'discord.Embed', ([], {'title': 'f"""Page {count}"""', 'description': 'chunk'}), "(title=f'Page {count}', description=chunk)\n", (23212, 23254), False, 'import discord\n'), ((24081, 24099), 'llama_index.GoogleDocsReader', 'GoogleDocsReader', ([], {}), '()\n', (24097, 24099), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((27234, 27277), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', ([], {'input_files': '[f.name]'}), '(input_files=[f.name])\n', (27255, 27277), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((27543, 27566), 'aiohttp.ClientSession', 'aiohttp.ClientSession', ([], {}), '()\n', (27564, 27566), False, 'import aiohttp\n'), ((28806, 28827), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (28825, 28827), False, 'import traceback\n'), ((28904, 28971), 'llama_index.BeautifulSoupWebReader', 'BeautifulSoupWebReader', ([], {'website_extractor': 'DEFAULT_WEBSITE_EXTRACTOR'}), '(website_extractor=DEFAULT_WEBSITE_EXTRACTOR)\n', (28926, 28971), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((29186, 29313), 'functools.partial', 'functools.partial', (['GPTVectorStoreIndex.from_documents'], {'documents': 'documents', 'service_context': 'service_context', 'use_async': '(True)'}), '(GPTVectorStoreIndex.from_documents, documents=documents,\n    service_context=service_context, use_async=True)\n', (29203, 29313), False, 'import functools\n'), ((30898, 30936), 'aiofiles.tempfile.TemporaryDirectory', 'aiofiles.tempfile.TemporaryDirectory', ([], {}), '()\n', (30934, 30936), False, 'import aiofiles\n'), ((32352, 32373), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (32371, 32373), False, 'import traceback\n'), ((35560, 35581), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (35579, 35581), False, 'import traceback\n'), ((35733, 35754), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (35752, 35754), False, 'import traceback\n'), ((36629, 36662), 'models.check_model.UrlCheck.check_youtube_link', 'UrlCheck.check_youtube_link', (['link'], {}), '(link)\n', (36656, 36662), False, 'from models.check_model import UrlCheck\n'), ((39345, 39564), 'llama_index.langchain_helpers.agents.IndexToolConfig', 'IndexToolConfig', ([], {'query_engine': 'engine', 'name': 'f"""{link_cleaned}-index"""', 'description': 'f"""Use this tool if the query seems related to this summary: {summary}"""', 'tool_kwargs': "{'return_direct': False}", 'max_iterations': '(5)'}), "(query_engine=engine, name=f'{link_cleaned}-index',\n    description=\n    f'Use this tool if the query seems related to this summary: {summary}',\n    tool_kwargs={'return_direct': False}, max_iterations=5)\n", (39360, 39564), False, 'from llama_index.langchain_helpers.agents import IndexToolConfig, LlamaToolkit, create_llama_chat_agent, LlamaIndexTool\n'), ((39742, 39786), 'llama_index.langchain_helpers.agents.LlamaIndexTool.from_tool_config', 'LlamaIndexTool.from_tool_config', (['tool_config'], {}), '(tool_config)\n', (39773, 39786), False, 'from llama_index.langchain_helpers.agents import IndexToolConfig, LlamaToolkit, create_llama_chat_agent, LlamaIndexTool\n'), ((39934, 40324), 'langchain.agents.initialize_agent', 'initialize_agent', ([], {'tools': 'tools', 'llm': 'self.index_chat_chains[index_chat_ctx.channel.id].llm', 'agent': 'AgentType.OPENAI_FUNCTIONS', 'verbose': '(True)', 'agent_kwargs': 'self.index_chat_chains[index_chat_ctx.channel.id].agent_kwargs', 'memory': 'self.index_chat_chains[index_chat_ctx.channel.id].memory', 'handle_parsing_errors': '"""Check your output and make sure it conforms!"""', 'max_iterations': '(5)'}), "(tools=tools, llm=self.index_chat_chains[index_chat_ctx.\n    channel.id].llm, agent=AgentType.OPENAI_FUNCTIONS, verbose=True,\n    agent_kwargs=self.index_chat_chains[index_chat_ctx.channel.id].\n    agent_kwargs, memory=self.index_chat_chains[index_chat_ctx.channel.id].\n    memory, handle_parsing_errors=\n    'Check your output and make sure it conforms!', max_iterations=5)\n", (39950, 40324), False, 'from langchain.agents import initialize_agent, AgentType\n'), ((42978, 42999), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (42997, 42999), False, 'import traceback\n'), ((44522, 44543), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (44541, 44543), False, 'import traceback\n'), ((44939, 45001), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{ctx.guild.id}/{index}"""'], {}), "(f'indexes/{ctx.guild.id}/{index}')\n", (44966, 45001), False, 'from services.environment_service import EnvService\n'), ((45671, 45692), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (45690, 45692), False, 'import traceback\n'), ((46478, 46533), 'llama_index.readers.schema.base.Document', 'Document', ([], {'text': 'chunk_text', 'extra_info': 'document.metadata'}), '(text=chunk_text, extra_info=document.metadata)\n', (46486, 46533), False, 'from llama_index.readers.schema.base import Document\n'), ((46819, 46877), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}/{_index}"""'], {}), "(f'indexes/{user_id}/{_index}')\n", (46846, 46877), False, 'from services.environment_service import EnvService\n'), ((47269, 47318), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': '(0)', 'model_name': '"""gpt-4-32k"""'}), "(temperature=0, model_name='gpt-4-32k')\n", (47279, 47318), False, 'from langchain.chat_models import ChatOpenAI\n'), ((53270, 53291), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (53289, 53291), False, 'import traceback\n'), ((53791, 53834), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': '(0)', 'model_name': 'model'}), '(temperature=0, model_name=model)\n', (53801, 53834), False, 'from langchain.chat_models import ChatOpenAI\n'), ((56156, 56177), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (56175, 56177), False, 'import traceback\n'), ((59155, 59228), 'llama_index.readers.schema.base.Document', 'Document', ([], {'text': 'channel_content', 'extra_info': "{'channel_name': channel_name}"}), "(text=channel_content, extra_info={'channel_name': channel_name})\n", (59163, 59228), False, 'from llama_index.readers.schema.base import Document\n'), ((7102, 7114), 'datetime.date.today', 'date.today', ([], {}), '()\n', (7112, 7114), False, 'from datetime import date\n'), ((7123, 7135), 'datetime.date.today', 'date.today', ([], {}), '()\n', (7133, 7135), False, 'from datetime import date\n'), ((11916, 11935), 'pathlib.Path', 'Path', (['original_path'], {}), '(original_path)\n', (11920, 11935), False, 'from pathlib import Path\n'), ((16619, 16830), 'llama_index.langchain_helpers.agents.IndexToolConfig', 'IndexToolConfig', ([], {'query_engine': 'engine', 'name': 'f"""{filename}-index"""', 'description': 'f"""Use this tool if the query seems related to this summary: {summary}"""', 'tool_kwargs': "{'return_direct': False}", 'max_iterations': '(5)'}), "(query_engine=engine, name=f'{filename}-index', description=\n    f'Use this tool if the query seems related to this summary: {summary}',\n    tool_kwargs={'return_direct': False}, max_iterations=5)\n", (16634, 16830), False, 'from llama_index.langchain_helpers.agents import IndexToolConfig, LlamaToolkit, create_llama_chat_agent, LlamaIndexTool\n'), ((17048, 17092), 'llama_index.langchain_helpers.agents.LlamaIndexTool.from_tool_config', 'LlamaIndexTool.from_tool_config', (['tool_config'], {}), '(tool_config)\n', (17079, 17092), False, 'from llama_index.langchain_helpers.agents import IndexToolConfig, LlamaToolkit, create_llama_chat_agent, LlamaIndexTool\n'), ((17245, 17592), 'langchain.agents.initialize_agent', 'initialize_agent', ([], {'tools': 'tools', 'llm': 'self.index_chat_chains[message.channel.id].llm', 'agent': 'AgentType.OPENAI_FUNCTIONS', 'verbose': '(True)', 'agent_kwargs': 'self.index_chat_chains[message.channel.id].agent_kwargs', 'memory': 'self.index_chat_chains[message.channel.id].memory', 'handle_parsing_errors': '"""Check your output and make sure it conforms!"""'}), "(tools=tools, llm=self.index_chat_chains[message.channel.id\n    ].llm, agent=AgentType.OPENAI_FUNCTIONS, verbose=True, agent_kwargs=\n    self.index_chat_chains[message.channel.id].agent_kwargs, memory=self.\n    index_chat_chains[message.channel.id].memory, handle_parsing_errors=\n    'Check your output and make sure it conforms!')\n", (17261, 17592), False, 'from langchain.agents import initialize_agent, AgentType\n'), ((18841, 18890), 'models.embed_statics_model.EmbedStatics.get_index_chat_preparation_message', 'EmbedStatics.get_index_chat_preparation_message', ([], {}), '()\n', (18888, 18890), False, 'from models.embed_statics_model import EmbedStatics\n'), ((24914, 24939), 'llama_index.readers.YoutubeTranscriptReader', 'YoutubeTranscriptReader', ([], {}), '()\n', (24937, 24939), False, 'from llama_index.readers import YoutubeTranscriptReader\n'), ((25561, 25607), 'llama_index.GithubRepositoryReader', 'GithubRepositoryReader', ([], {'owner': 'owner', 'repo': 'repo'}), '(owner=owner, repo=repo)\n', (25583, 25607), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((26838, 26894), 'tempfile.NamedTemporaryFile', 'tempfile.NamedTemporaryFile', ([], {'suffix': '""".pdf"""', 'delete': '(False)'}), "(suffix='.pdf', delete=False)\n", (26865, 26894), False, 'import tempfile\n'), ((30978, 31063), 'aiofiles.tempfile.NamedTemporaryFile', 'aiofiles.tempfile.NamedTemporaryFile', ([], {'suffix': 'suffix', 'dir': 'temp_path', 'delete': '(False)'}), '(suffix=suffix, dir=temp_path, delete=False\n    )\n', (31014, 31063), False, 'import aiofiles\n'), ((31886, 31907), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (31905, 31907), False, 'import traceback\n'), ((32754, 32795), 'models.embed_statics_model.EmbedStatics.build_index_progress_embed', 'EmbedStatics.build_index_progress_embed', ([], {}), '()\n', (32793, 32795), False, 'from models.embed_statics_model import EmbedStatics\n'), ((32931, 32954), 'aiohttp.ClientSession', 'aiohttp.ClientSession', ([], {}), '()\n', (32952, 32954), False, 'import aiohttp\n'), ((33623, 33644), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (33642, 33644), False, 'import traceback\n'), ((34114, 34147), 'functools.partial', 'partial', (['loader.load_data', '[link]'], {}), '(loader.load_data, [link])\n', (34121, 34147), False, 'from functools import partial\n'), ((34253, 34373), 'functools.partial', 'functools.partial', (['GPTVectorStoreIndex'], {'documents': 'documents', 'service_context': 'service_context_no_llm', 'use_async': '(True)'}), '(GPTVectorStoreIndex, documents=documents, service_context\n    =service_context_no_llm, use_async=True)\n', (34270, 34373), False, 'import functools\n'), ((34827, 34848), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (34846, 34848), False, 'import traceback\n'), ((35809, 35856), 'models.embed_statics_model.EmbedStatics.get_index_set_success_embed', 'EmbedStatics.get_index_set_success_embed', (['price'], {}), '(price)\n', (35849, 35856), False, 'from models.embed_statics_model import EmbedStatics\n'), ((41745, 41786), 'models.embed_statics_model.EmbedStatics.build_index_progress_embed', 'EmbedStatics.build_index_progress_embed', ([], {}), '()\n', (41784, 41786), False, 'from models.embed_statics_model import EmbedStatics\n'), ((42246, 42267), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (42265, 42267), False, 'import traceback\n'), ((43054, 43101), 'models.embed_statics_model.EmbedStatics.get_index_set_success_embed', 'EmbedStatics.get_index_set_success_embed', (['price'], {}), '(price)\n', (43094, 43101), False, 'from models.embed_statics_model import EmbedStatics\n'), ((43736, 43797), 'functools.partial', 'partial', (['self.index_discord', 'document', 'service_context_no_llm'], {}), '(self.index_discord, document, service_context_no_llm)\n', (43743, 43797), False, 'from functools import partial\n'), ((44028, 44049), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (44047, 44049), False, 'import traceback\n'), ((45094, 45162), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{ctx.user.id}_search/{index}"""'], {}), "(f'indexes/{ctx.user.id}_search/{index}')\n", (45121, 45162), False, 'from services.environment_service import EnvService\n'), ((45248, 45309), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{ctx.user.id}/{index}"""'], {}), "(f'indexes/{ctx.user.id}/{index}')\n", (45275, 45309), False, 'from services.environment_service import EnvService\n'), ((45423, 45464), 'functools.partial', 'partial', (['self.index_load_file', 'index_file'], {}), '(self.index_load_file, index_file)\n', (45430, 45464), False, 'from functools import partial\n'), ((46938, 47003), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}_search/{_index}"""'], {}), "(f'indexes/{user_id}_search/{_index}')\n", (46965, 47003), False, 'from services.environment_service import EnvService\n'), ((47118, 47159), 'functools.partial', 'partial', (['self.index_load_file', 'index_file'], {}), '(self.index_load_file, index_file)\n', (47125, 47159), False, 'from functools import partial\n'), ((48341, 48441), 'functools.partial', 'partial', (['GPTTreeIndex.from_documents'], {'documents': 'documents', 'service_context': 'service_context_mock'}), '(GPTTreeIndex.from_documents, documents=documents, service_context=\n    service_context_mock)\n', (48348, 48441), False, 'from functools import partial\n'), ((49269, 49385), 'functools.partial', 'partial', (['GPTTreeIndex.from_documents'], {'documents': 'documents', 'service_context': 'self.service_context', 'use_async': '(True)'}), '(GPTTreeIndex.from_documents, documents=documents, service_context=\n    self.service_context, use_async=True)\n', (49276, 49385), False, 'from functools import partial\n'), ((50492, 50616), 'functools.partial', 'partial', (['GPTVectorStoreIndex.from_documents'], {'documents': 'documents', 'service_context': 'service_context_no_llm', 'use_async': '(True)'}), '(GPTVectorStoreIndex.from_documents, documents=documents,\n    service_context=service_context_no_llm, use_async=True)\n', (50499, 50616), False, 'from functools import partial\n'), ((52163, 52224), 'functools.partial', 'partial', (['self.index_discord', 'document', 'service_context_no_llm'], {}), '(self.index_discord, document, service_context_no_llm)\n', (52170, 52224), False, 'from functools import partial\n'), ((52591, 52612), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (52610, 52612), False, 'import traceback\n'), ((53897, 53949), 'models.embed_statics_model.EmbedStatics.build_index_query_progress_embed', 'EmbedStatics.build_index_query_progress_embed', (['query'], {}), '(query)\n', (53942, 53949), False, 'from models.embed_statics_model import EmbedStatics\n'), ((54109, 54311), 'functools.partial', 'partial', (['get_and_query', 'ctx.user.id', 'self.index_storage', 'query', 'response_mode', 'nodes', 'child_branch_factor'], {'service_context': 'service_context_no_llm', 'multistep': '(llm_predictor if multistep else None)'}), '(get_and_query, ctx.user.id, self.index_storage, query,\n    response_mode, nodes, child_branch_factor, service_context=\n    service_context_no_llm, multistep=llm_predictor if multistep else None)\n', (54116, 54311), False, 'from functools import partial\n'), ((63481, 63527), 'discord.SelectOption', 'discord.SelectOption', ([], {'label': '"""Yes"""', 'value': '"""yes"""'}), "(label='Yes', value='yes')\n", (63501, 63527), False, 'import discord\n'), ((63545, 63589), 'discord.SelectOption', 'discord.SelectOption', ([], {'label': '"""No"""', 'value': '"""no"""'}), "(label='No', value='no')\n", (63565, 63589), False, 'import discord\n'), ((6286, 6335), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}"""'], {}), "(f'indexes/{user_id}')\n", (6313, 6335), False, 'from services.environment_service import EnvService\n'), ((6537, 6593), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}_search"""'], {}), "(f'indexes/{user_id}_search')\n", (6564, 6593), False, 'from services.environment_service import EnvService\n'), ((7790, 7846), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}/{file}"""'], {}), "(f'indexes/{user_id}/{file}')\n", (7817, 7846), False, 'from services.environment_service import EnvService\n'), ((7892, 7913), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (7911, 7913), False, 'import traceback\n'), ((8114, 8177), 'services.environment_service.EnvService.find_shared_file', 'EnvService.find_shared_file', (['f"""indexes/{user_id}_search/{file}"""'], {}), "(f'indexes/{user_id}_search/{file}')\n", (8141, 8177), False, 'from services.environment_service import EnvService\n'), ((8244, 8265), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (8263, 8265), False, 'import traceback\n'), ((18637, 18658), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (18656, 18658), False, 'import traceback\n'), ((23775, 23821), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', ([], {'input_files': '[file_path]'}), '(input_files=[file_path])\n', (23796, 23821), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((25712, 25758), 'llama_index.GithubRepositoryReader', 'GithubRepositoryReader', ([], {'owner': 'owner', 'repo': 'repo'}), '(owner=owner, repo=repo)\n', (25734, 25758), False, 'from llama_index import GPTVectorStoreIndex, SimpleDirectoryReader, QuestionAnswerPrompt, BeautifulSoupWebReader, GPTTreeIndex, GoogleDocsReader, MockLLMPredictor, OpenAIEmbedding, GithubRepositoryReader, MockEmbedding, download_loader, LLMPredictor, ServiceContext, StorageContext, load_index_from_storage, get_response_synthesizer, VectorStoreIndex\n'), ((30818, 30859), 'models.embed_statics_model.EmbedStatics.build_index_progress_embed', 'EmbedStatics.build_index_progress_embed', ([], {}), '()\n', (30857, 30859), False, 'from models.embed_statics_model import EmbedStatics\n'), ((36820, 36888), 'functools.partial', 'partial', (['self.index_youtube_transcript', 'link', 'service_context_no_llm'], {}), '(self.index_youtube_transcript, link, service_context_no_llm)\n', (36827, 36888), False, 'from functools import partial\n'), ((44344, 44391), 'models.embed_statics_model.EmbedStatics.get_index_set_success_embed', 'EmbedStatics.get_index_set_success_embed', (['price'], {}), '(price)\n', (44384, 44391), False, 'from models.embed_statics_model import EmbedStatics\n'), ((45583, 45626), 'models.embed_statics_model.EmbedStatics.get_index_load_success_embed', 'EmbedStatics.get_index_load_success_embed', ([], {}), '()\n', (45624, 45626), False, 'from models.embed_statics_model import EmbedStatics\n'), ((47807, 47854), 'tiktoken.encoding_for_model', 'tiktoken.encoding_for_model', (['"""text-davinci-003"""'], {}), "('text-davinci-003')\n", (47834, 47854), False, 'import tiktoken\n'), ((53090, 53137), 'models.embed_statics_model.EmbedStatics.get_index_set_success_embed', 'EmbedStatics.get_index_set_success_embed', (['price'], {}), '(price)\n', (53130, 53137), False, 'from models.embed_statics_model import EmbedStatics\n'), ((55986, 56050), 'models.embed_statics_model.EmbedStatics.build_index_query_success_embed', 'EmbedStatics.build_index_query_success_embed', (['query', 'total_price'], {}), '(query, total_price)\n', (56030, 56050), False, 'from models.embed_statics_model import EmbedStatics\n'), ((59758, 59855), 'models.embed_statics_model.EmbedStatics.get_index_compose_failure_embed', 'EmbedStatics.get_index_compose_failure_embed', (['"""You must have at least one index to compose."""'], {}), "(\n    'You must have at least one index to compose.')\n", (59802, 59855), False, 'from models.embed_statics_model import EmbedStatics\n'), ((61269, 61292), 'random.randint', 'random.randint', (['(0)', '(9999)'], {}), '(0, 9999)\n', (61283, 61292), False, 'import random\n'), ((61987, 62080), 'discord.SelectOption', 'discord.SelectOption', ([], {'label': 'LONG_TO_SHORT_CACHE[index]', 'value': 'LONG_TO_SHORT_CACHE[index]'}), '(label=LONG_TO_SHORT_CACHE[index], value=\n    LONG_TO_SHORT_CACHE[index])\n', (62007, 62080), False, 'import discord\n'), ((65878, 65899), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (65897, 65899), False, 'import traceback\n'), ((67068, 67089), 'traceback.print_exc', 'traceback.print_exc', ([], {}), '()\n', (67087, 67089), False, 'import traceback\n'), ((6932, 6954), 'services.environment_service.EnvService.save_path', 'EnvService.save_path', ([], {}), '()\n', (6952, 6954), False, 'from services.environment_service import EnvService\n'), ((7328, 7350), 'services.environment_service.EnvService.save_path', 'EnvService.save_path', ([], {}), '()\n', (7348, 7350), False, 'from services.environment_service import EnvService\n'), ((30615, 30675), 'models.embed_statics_model.EmbedStatics.get_index_set_failure_embed', 'EmbedStatics.get_index_set_failure_embed', (['"""Unsupported file"""'], {}), "('Unsupported file')\n", (30655, 30675), False, 'from models.embed_statics_model import EmbedStatics\n'), ((37144, 37211), 'functools.partial', 'partial', (['self.index_github_repository', 'link', 'service_context_no_llm'], {}), '(self.index_github_repository, link, service_context_no_llm)\n', (37151, 37211), False, 'from functools import partial\n'), ((49890, 49902), 'datetime.date.today', 'date.today', ([], {}), '()\n', (49900, 49902), False, 'from datetime import date\n'), ((49911, 49923), 'datetime.date.today', 'date.today', ([], {}), '()\n', (49921, 49923), False, 'from datetime import date\n'), ((50916, 50928), 'datetime.date.today', 'date.today', ([], {}), '()\n', (50926, 50928), False, 'from datetime import date\n'), ((50937, 50949), 'datetime.date.today', 'date.today', ([], {}), '()\n', (50947, 50949), False, 'from datetime import date\n'), ((56237, 56376), 'models.embed_statics_model.EmbedStatics.get_index_query_failure_embed', 'EmbedStatics.get_index_query_failure_embed', (['"""Failed to send query. You may not have an index set, load an index with /index load"""'], {}), "(\n    'Failed to send query. You may not have an index set, load an index with /index load'\n    )\n", (56279, 56376), False, 'from models.embed_statics_model import EmbedStatics\n'), ((64697, 64782), 'models.embed_statics_model.EmbedStatics.get_index_compose_failure_embed', 'EmbedStatics.get_index_compose_failure_embed', (['"""You must select at least 1 index"""'], {}), "('You must select at least 1 index'\n    )\n", (64741, 64782), False, 'from models.embed_statics_model import EmbedStatics\n'), ((65000, 65047), 'models.embed_statics_model.EmbedStatics.get_index_compose_progress_embed', 'EmbedStatics.get_index_compose_progress_embed', ([], {}), '()\n', (65045, 65047), False, 'from models.embed_statics_model import EmbedStatics\n'), ((66356, 66407), 'models.embed_statics_model.EmbedStatics.get_index_compose_success_embed', 'EmbedStatics.get_index_compose_success_embed', (['price'], {}), '(price)\n', (66400, 66407), False, 'from models.embed_statics_model import EmbedStatics\n'), ((14362, 14382), 'pathlib.Path', 'Path', (['temp_file.name'], {}), '(temp_file.name)\n', (14366, 14382), False, 'from pathlib import Path\n'), ((31360, 31380), 'pathlib.Path', 'Path', (['temp_file.name'], {}), '(temp_file.name)\n', (31364, 31380), False, 'from pathlib import Path\n'), ((50065, 50087), 'services.environment_service.EnvService.save_path', 'EnvService.save_path', ([], {}), '()\n', (50085, 50087), False, 'from services.environment_service import EnvService\n'), ((51088, 51110), 'services.environment_service.EnvService.save_path', 'EnvService.save_path', ([], {}), '()\n', (51108, 51110), False, 'from services.environment_service import EnvService\n'), ((52664, 52686), 'services.environment_service.EnvService.save_path', 'EnvService.save_path', ([], {}), '()\n', (52684, 52686), False, 'from services.environment_service import EnvService\n'), ((52856, 52878), 'services.environment_service.EnvService.save_path', 'EnvService.save_path', ([], {}), '()\n', (52876, 52878), False, 'from services.environment_service import EnvService\n'), ((28402, 28529), 'functools.partial', 'functools.partial', (['GPTVectorStoreIndex.from_documents'], {'documents': 'documents', 'service_context': 'service_context', 'use_async': '(True)'}), '(GPTVectorStoreIndex.from_documents, documents=documents,\n    service_context=service_context, use_async=True)\n', (28419, 28529), False, 'import functools\n'), ((52999, 53011), 'datetime.date.today', 'date.today', ([], {}), '()\n', (53009, 53011), False, 'from datetime import date\n'), ((53020, 53032), 'datetime.date.today', 'date.today', ([], {}), '()\n', (53030, 53032), False, 'from datetime import date\n'), ((62799, 62892), 'discord.SelectOption', 'discord.SelectOption', ([], {'label': 'LONG_TO_SHORT_CACHE[index]', 'value': 'LONG_TO_SHORT_CACHE[index]'}), '(label=LONG_TO_SHORT_CACHE[index], value=\n    LONG_TO_SHORT_CACHE[index])\n', (62819, 62892), False, 'import discord\n'), ((33339, 33441), 'models.embed_statics_model.EmbedStatics.get_index_set_failure_embed', 'EmbedStatics.get_index_set_failure_embed', (['"""Invalid URL or could not connect to the provided URL."""'], {}), "(\n    'Invalid URL or could not connect to the provided URL.')\n", (33379, 33441), False, 'from models.embed_statics_model import EmbedStatics\n')]