code: import os
  import openai
  import tiktoken
  import logging
  import sys
  from dotenv import load_dotenv
  from threading import Lock
  from llama_index import (SimpleDirectoryReader,
                           VectorStoreIndex,
                           ServiceContext,
                           StorageContext,
                           load_index_from_storage,
                           set_global_service_context,
                           download_loader)
  from llama_hub.github_repo import GithubRepositoryReader, GithubClient
  from llama_index.callbacks import CallbackManager, TokenCountingHandler
  from llama_index.llms import OpenAI
  from base_prompt import CHAT_TEXT_QA_PROMPT, CHAT_REFINE_PROMPT
  from llama_index.indices.postprocessor import SentenceTransformerRerank
  from llama_index.query_engine.router_query_engine import RouterQueryEngine
  from llama_index.selectors.pydantic_selectors import PydanticSingleSelector
  from llama_index.tools.query_engine import QueryEngineTool
  
  logging.basicConfig(stream=sys.stdout, level=logging.INFO)
  logging.getLogger().handlers = []
  logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
  
  load_dotenv()
  openai.api_key = os.environ["OPENAI_API_KEY"]
  GITHUB_API_KEY = os.environ["GITHUB_API_KEY"]
  thread_lock = Lock()
  
  # setup token counter
  token_counter = TokenCountingHandler(tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode)
  callback_manager = CallbackManager([token_counter])
  # define LLM
  llm = OpenAI(temperature=0, model="gpt-3.5-turbo", streaming=False, max_tokens=1000)
  service_context = ServiceContext.from_defaults(llm=llm, callback_manager=callback_manager, embed_model="local")
  set_global_service_context(service_context)
  
  
  # builds new index from our data folder and GitHub repos
  def build_index():
      global service_context
  
      general_documents = []
      code_documents = []
      # load directory documents
      directory_document = SimpleDirectoryReader('./data', recursive=True).load_data()
      general_documents += directory_document
      # load github documents
      download_loader("GithubRepositoryReader")
      github_client = GithubClient(GITHUB_API_KEY)
      owner = "pyth-network"
      repos = ["pyth-client-py", "pyth-client-js", "pyth-sdk-solidity", "pyth-sdk-rs", "documentation", "pyth-crosschain"]
      branch = "main"
      # build documents out of all repos
      for repo in repos:
          if repo == "documentation":
              loader = GithubRepositoryReader(github_client,
                                              owner=owner,
                                              repo=repo,
                                              filter_directories=(["images"], GithubRepositoryReader.FilterType.EXCLUDE),
                                              verbose=False,
                                              concurrent_requests=10,
                                              )
              document = loader.load_data(branch=branch)
              general_documents += document
          else:
              loader = GithubRepositoryReader(github_client,
                                              owner=owner,
                                              repo=repo,
                                              filter_directories=(["images"], GithubRepositoryReader.FilterType.EXCLUDE),
                                              verbose=False,
                                              concurrent_requests=10,
                                              )
              document = loader.load_data(branch=branch)
              code_documents += document
      # build index and store it
      general_index = VectorStoreIndex.from_documents(general_documents, service_context=service_context, show_progress=True)
      general_index.storage_context.persist(persist_dir="./storage/general_index")
  
      code_index = VectorStoreIndex.from_documents(code_documents, service_context=service_context, show_progress=True)
      code_index.storage_context.persist(persist_dir="./storage/code_index")
  
  
  # used to add documents to existing stored index
  def add_to_index():
      download_loader("GithubRepositoryReader")
      github_client = GithubClient(GITHUB_API_KEY)
      owner = "pyth-network"
      repos = ["pyth-serum", "publisher-utils", "solmeet-workshop-june-22", "oracle-sandbox", "pyth-sdk-js",
               "program-authority-escrow", "pyth-observer", "audit-reports", "example-publisher", "pyth-agent",
               "program-admin", "pyth-client", "pythnet", "governance"]
      branch = "main"
  
      combined_documents = []
      for repo in repos:
          loader = GithubRepositoryReader(github_client, owner=owner, repo=repo, filter_directories=(["images"], GithubRepositoryReader.FilterType.EXCLUDE), verbose=False, concurrent_requests=10)
          if repo == "governance":
              document = loader.load_data(branch="master")
          elif repo == "pythnet":
              document = loader.load_data(branch="pyth")
          else:
              document = loader.load_data(branch=branch)
          combined_documents += document
  
      storage_context = StorageContext.from_defaults(persist_dir="./storage")
      index = load_index_from_storage(storage_context)
      for doc in combined_documents:
          index.insert(doc)
      index.storage_context.persist()
  
  
  def pyth_gpt(message):
      global service_context
  
      with thread_lock:
          # rebuild storage context
          general_storage_context = StorageContext.from_defaults(persist_dir="./storage/general_index")
          code_storage_context = StorageContext.from_defaults(persist_dir="./storage/code_index")
          # load index
          general_index = load_index_from_storage(general_storage_context, service_context=service_context)
          code_index = load_index_from_storage(code_storage_context, service_context=service_context)
  
          rerank = SentenceTransformerRerank(model="cross-encoder/ms-marco-MiniLM-L-2-v2", top_n=3)
          # query the index
          general_query_engine = general_index.as_query_engine(text_qa_template=CHAT_TEXT_QA_PROMPT,
                                                               refine_template=CHAT_REFINE_PROMPT,
                                                               similarity_top_k=10,
                                                               streaming=False,
                                                               service_context=service_context,
                                                               node_postprocessors=[rerank])
  
          code_query_engine = code_index.as_query_engine(text_qa_template=CHAT_TEXT_QA_PROMPT,
                                                         refine_template=CHAT_REFINE_PROMPT,
                                                         similarity_top_k=10,
                                                         streaming=False,
                                                         service_context=service_context,
                                                         node_postprocessors=[rerank])
  
          general_vector_tool = QueryEngineTool.from_defaults(query_engine=general_query_engine, description="Useful for retrieving general context related to the data source", )
          code_vector_tool = QueryEngineTool.from_defaults(query_engine=code_query_engine, description="Useful specifically for coding questions related to the data source ", )
  
          query_engine = RouterQueryEngine(selector=PydanticSingleSelector.from_defaults(),
                                           query_engine_tools=[general_vector_tool, code_vector_tool])
          # enter your prompt
          response = query_engine.query(message)
          # token counter
          print('Embedding Tokens: ', token_counter.total_embedding_token_count, '\n',
                'LLM Prompt Tokens: ', token_counter.prompt_llm_token_count, '\n',
                'LLM Completion Tokens: ', token_counter.completion_llm_token_count, '\n',
                'Total LLM Token Count: ', token_counter.total_llm_token_count, '\n')
          token_counter.reset_counts()
  
          return str(response)
  
apis: llama_index.VectorStoreIndex.from_documents llama_index.tools.query_engine.QueryEngineTool.from_defaults llama_index.ServiceContext.from_defaults llama_index.StorageContext.from_defaults llama_index.selectors.pydantic_selectors.PydanticSingleSelector.from_defaults llama_index.SimpleDirectoryReader llama_index.llms.OpenAI llama_index.download_loader llama_index.callbacks.CallbackManager llama_index.load_index_from_storage llama_index.set_global_service_context llama_index.indices.postprocessor.SentenceTransformerRerank
extract_api: [((991, 1049), 'logging.basicConfig', 'logging.basicConfig', ([], {'stream': 'sys.stdout', 'level': 'logging.INFO'}), '(stream=sys.stdout, level=logging.INFO)\n', (1010, 1049), False, 'import logging\n'), ((1162, 1175), 'dotenv.load_dotenv', 'load_dotenv', ([], {}), '()\n', (1173, 1175), False, 'from dotenv import load_dotenv\n'), ((1285, 1291), 'threading.Lock', 'Lock', ([], {}), '()\n', (1289, 1291), False, 'from threading import Lock\n'), ((1438, 1470), 'llama_index.callbacks.CallbackManager', 'CallbackManager', (['[token_counter]'], {}), '([token_counter])\n', (1453, 1470), False, 'from llama_index.callbacks import CallbackManager, TokenCountingHandler\n'), ((1492, 1570), 'llama_index.llms.OpenAI', 'OpenAI', ([], {'temperature': '(0)', 'model': '"""gpt-3.5-turbo"""', 'streaming': '(False)', 'max_tokens': '(1000)'}), "(temperature=0, model='gpt-3.5-turbo', streaming=False, max_tokens=1000)\n", (1498, 1570), False, 'from llama_index.llms import OpenAI\n'), ((1590, 1687), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'llm': 'llm', 'callback_manager': 'callback_manager', 'embed_model': '"""local"""'}), "(llm=llm, callback_manager=callback_manager,\n    embed_model='local')\n", (1618, 1687), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((1685, 1728), 'llama_index.set_global_service_context', 'set_global_service_context', (['service_context'], {}), '(service_context)\n', (1711, 1728), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((1051, 1070), 'logging.getLogger', 'logging.getLogger', ([], {}), '()\n', (1068, 1070), False, 'import logging\n'), ((1117, 1157), 'logging.StreamHandler', 'logging.StreamHandler', ([], {'stream': 'sys.stdout'}), '(stream=sys.stdout)\n', (1138, 1157), False, 'import logging\n'), ((2091, 2132), 'llama_index.download_loader', 'download_loader', (['"""GithubRepositoryReader"""'], {}), "('GithubRepositoryReader')\n", (2106, 2132), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((2154, 2182), 'llama_hub.github_repo.GithubClient', 'GithubClient', (['GITHUB_API_KEY'], {}), '(GITHUB_API_KEY)\n', (2166, 2182), False, 'from llama_hub.github_repo import GithubRepositoryReader, GithubClient\n'), ((3660, 3768), 'llama_index.VectorStoreIndex.from_documents', 'VectorStoreIndex.from_documents', (['general_documents'], {'service_context': 'service_context', 'show_progress': '(True)'}), '(general_documents, service_context=\n    service_context, show_progress=True)\n', (3691, 3768), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((3866, 3971), 'llama_index.VectorStoreIndex.from_documents', 'VectorStoreIndex.from_documents', (['code_documents'], {'service_context': 'service_context', 'show_progress': '(True)'}), '(code_documents, service_context=\n    service_context, show_progress=True)\n', (3897, 3971), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((4123, 4164), 'llama_index.download_loader', 'download_loader', (['"""GithubRepositoryReader"""'], {}), "('GithubRepositoryReader')\n", (4138, 4164), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((4186, 4214), 'llama_hub.github_repo.GithubClient', 'GithubClient', (['GITHUB_API_KEY'], {}), '(GITHUB_API_KEY)\n', (4198, 4214), False, 'from llama_hub.github_repo import GithubRepositoryReader, GithubClient\n'), ((5121, 5174), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': '"""./storage"""'}), "(persist_dir='./storage')\n", (5149, 5174), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((5188, 5228), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['storage_context'], {}), '(storage_context)\n', (5211, 5228), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((1086, 1105), 'logging.getLogger', 'logging.getLogger', ([], {}), '()\n', (1103, 1105), False, 'import logging\n'), ((4627, 4812), 'llama_hub.github_repo.GithubRepositoryReader', 'GithubRepositoryReader', (['github_client'], {'owner': 'owner', 'repo': 'repo', 'filter_directories': "(['images'], GithubRepositoryReader.FilterType.EXCLUDE)", 'verbose': '(False)', 'concurrent_requests': '(10)'}), "(github_client, owner=owner, repo=repo,\n    filter_directories=(['images'], GithubRepositoryReader.FilterType.\n    EXCLUDE), verbose=False, concurrent_requests=10)\n", (4649, 4812), False, 'from llama_hub.github_repo import GithubRepositoryReader, GithubClient\n'), ((5480, 5547), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': '"""./storage/general_index"""'}), "(persist_dir='./storage/general_index')\n", (5508, 5547), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((5580, 5644), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {'persist_dir': '"""./storage/code_index"""'}), "(persist_dir='./storage/code_index')\n", (5608, 5644), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((5692, 5778), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['general_storage_context'], {'service_context': 'service_context'}), '(general_storage_context, service_context=\n    service_context)\n', (5715, 5778), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((5796, 5874), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['code_storage_context'], {'service_context': 'service_context'}), '(code_storage_context, service_context=service_context)\n', (5819, 5874), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((5895, 5980), 'llama_index.indices.postprocessor.SentenceTransformerRerank', 'SentenceTransformerRerank', ([], {'model': '"""cross-encoder/ms-marco-MiniLM-L-2-v2"""', 'top_n': '(3)'}), "(model='cross-encoder/ms-marco-MiniLM-L-2-v2', top_n=3\n    )\n", (5920, 5980), False, 'from llama_index.indices.postprocessor import SentenceTransformerRerank\n'), ((7096, 7249), 'llama_index.tools.query_engine.QueryEngineTool.from_defaults', 'QueryEngineTool.from_defaults', ([], {'query_engine': 'general_query_engine', 'description': '"""Useful for retrieving general context related to the data source"""'}), "(query_engine=general_query_engine,\n    description=\n    'Useful for retrieving general context related to the data source')\n", (7125, 7249), False, 'from llama_index.tools.query_engine import QueryEngineTool\n'), ((7271, 7421), 'llama_index.tools.query_engine.QueryEngineTool.from_defaults', 'QueryEngineTool.from_defaults', ([], {'query_engine': 'code_query_engine', 'description': '"""Useful specifically for coding questions related to the data source """'}), "(query_engine=code_query_engine, description=\n    'Useful specifically for coding questions related to the data source ')\n", (7300, 7421), False, 'from llama_index.tools.query_engine import QueryEngineTool\n'), ((1365, 1409), 'tiktoken.encoding_for_model', 'tiktoken.encoding_for_model', (['"""gpt-3.5-turbo"""'], {}), "('gpt-3.5-turbo')\n", (1392, 1409), False, 'import tiktoken\n'), ((1952, 1999), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', (['"""./data"""'], {'recursive': '(True)'}), "('./data', recursive=True)\n", (1973, 1999), False, 'from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext, StorageContext, load_index_from_storage, set_global_service_context, download_loader\n'), ((2477, 2662), 'llama_hub.github_repo.GithubRepositoryReader', 'GithubRepositoryReader', (['github_client'], {'owner': 'owner', 'repo': 'repo', 'filter_directories': "(['images'], GithubRepositoryReader.FilterType.EXCLUDE)", 'verbose': '(False)', 'concurrent_requests': '(10)'}), "(github_client, owner=owner, repo=repo,\n    filter_directories=(['images'], GithubRepositoryReader.FilterType.\n    EXCLUDE), verbose=False, concurrent_requests=10)\n", (2499, 2662), False, 'from llama_hub.github_repo import GithubRepositoryReader, GithubClient\n'), ((3062, 3247), 'llama_hub.github_repo.GithubRepositoryReader', 'GithubRepositoryReader', (['github_client'], {'owner': 'owner', 'repo': 'repo', 'filter_directories': "(['images'], GithubRepositoryReader.FilterType.EXCLUDE)", 'verbose': '(False)', 'concurrent_requests': '(10)'}), "(github_client, owner=owner, repo=repo,\n    filter_directories=(['images'], GithubRepositoryReader.FilterType.\n    EXCLUDE), verbose=False, concurrent_requests=10)\n", (3084, 3247), False, 'from llama_hub.github_repo import GithubRepositoryReader, GithubClient\n'), ((7472, 7510), 'llama_index.selectors.pydantic_selectors.PydanticSingleSelector.from_defaults', 'PydanticSingleSelector.from_defaults', ([], {}), '()\n', (7508, 7510), False, 'from llama_index.selectors.pydantic_selectors import PydanticSingleSelector\n')]