code: 
  import os
  import logging
  import hashlib
  import random
  import uuid
  import openai
  from pathlib import Path
  from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage
  from llama_index.readers.schema.base import Document
  from langchain.chat_models import ChatOpenAI
  from azure.cognitiveservices.speech import SpeechConfig, SpeechSynthesizer, ResultReason, CancellationReason, SpeechSynthesisOutputFormat
  from azure.cognitiveservices.speech.audio import AudioOutputConfig
  
  from app.fetch_web_post import get_urls, get_youtube_transcript, scrape_website, scrape_website_by_phantomjscloud
  from app.prompt import get_prompt_template
  from app.util import get_language_code, get_youtube_video_id
  
  OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
  SPEECH_KEY = os.environ.get('SPEECH_KEY')
  SPEECH_REGION = os.environ.get('SPEECH_REGION')
  openai.api_key = OPENAI_API_KEY
  
  index_cache_web_dir = Path('/tmp/myGPTReader/cache_web/')
  index_cache_file_dir = Path('/data/myGPTReader/file/')
  index_cache_voice_dir = Path('/tmp/myGPTReader/voice/')
  
  if not index_cache_web_dir.is_dir():
      index_cache_web_dir.mkdir(parents=True, exist_ok=True)
  
  if not index_cache_voice_dir.is_dir():
      index_cache_voice_dir.mkdir(parents=True, exist_ok=True)
  
  if not index_cache_file_dir.is_dir():
      index_cache_file_dir.mkdir(parents=True, exist_ok=True)
  
  llm_predictor = LLMPredictor(llm=ChatOpenAI(
      temperature=0, model_name="gpt-3.5-turbo"))
  
  service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)
  
  web_storage_context = StorageContext.from_defaults()
  file_storage_context = StorageContext.from_defaults()
  
  def get_unique_md5(urls):
      urls_str = ''.join(sorted(urls))
      hashed_str = hashlib.md5(urls_str.encode('utf-8')).hexdigest()
      return hashed_str
  
  def format_dialog_messages(messages):
      return "\n".join(messages)
  
  def get_document_from_youtube_id(video_id):
      if video_id is None:
          return None
      transcript = get_youtube_transcript(video_id)
      if transcript is None:
          return None
      return Document(transcript)
  
  def remove_prompt_from_text(text):
      return text.replace('chatGPT:', '').strip()
  
  def get_documents_from_urls(urls):
      documents = []
      for url in urls['page_urls']:
          document = Document(scrape_website(url))
          documents.append(document)
      if len(urls['rss_urls']) > 0:
          rss_documents = RssReader().load_data(urls['rss_urls'])
          documents = documents + rss_documents
      if len(urls['phantomjscloud_urls']) > 0:
          for url in urls['phantomjscloud_urls']:
              document = Document(scrape_website_by_phantomjscloud(url))
              documents.append(document)
      if len(urls['youtube_urls']) > 0:
          for url in urls['youtube_urls']:
              video_id = get_youtube_video_id(url)
              document = get_document_from_youtube_id(video_id)
              if (document is not None):
                  documents.append(document)
              else:
                  documents.append(Document(f"Can't get transcript from youtube video: {url}"))
      return documents
  
  def get_index_from_web_cache(name):
      try:
          index = load_index_from_storage(web_storage_context, index_id=name)
      except Exception as e:
          logging.error(e)
          return None
      return index
  
  def get_index_from_file_cache(name):
      try:
          index = load_index_from_storage(file_storage_context, index_id=name)
      except Exception as e:
          logging.error(e)
          return None
      return index
  
  def get_index_name_from_file(file: str):
      file_md5_with_extension = str(Path(file).relative_to(index_cache_file_dir).name)
      file_md5 = file_md5_with_extension.split('.')[0]
      return file_md5
  
  def get_answer_from_chatGPT(messages):
      dialog_messages = format_dialog_messages(messages)
      logging.info('=====> Use chatGPT to answer!')
      logging.info(dialog_messages)
      completion = openai.ChatCompletion.create(
          model="gpt-3.5-turbo",
          messages=[{"role": "user", "content": dialog_messages}]
      )
      logging.info(completion.usage)
      total_tokens = completion.usage.total_tokens
      return completion.choices[0].message.content, total_tokens, None
  
  def get_answer_from_llama_web(messages, urls):
      dialog_messages = format_dialog_messages(messages)
      lang_code = get_language_code(remove_prompt_from_text(messages[-1]))
      combained_urls = get_urls(urls)
      logging.info(combained_urls)
      index_file_name = get_unique_md5(urls)
      index = get_index_from_web_cache(index_file_name)
      if index is None:
          logging.info(f"=====> Build index from web!")
          documents = get_documents_from_urls(combained_urls)
          logging.info(documents)
          index = GPTVectorStoreIndex.from_documents(documents, service_context=service_context)
          index.set_index_id(index_file_name)
          index.storage_context.persist()
          logging.info(
              f"=====> Save index to disk path: {index_cache_web_dir / index_file_name}")
      prompt = get_prompt_template(lang_code)
      logging.info('=====> Use llama web with chatGPT to answer!')
      logging.info('=====> dialog_messages')
      logging.info(dialog_messages)
      logging.info('=====> text_qa_template')
      logging.info(prompt.prompt)
      answer = index.as_query_engine(text_qa_template=prompt).query(dialog_messages)
      total_llm_model_tokens = llm_predictor.last_token_usage
      total_embedding_model_tokens = service_context.embed_model.last_token_usage
      return answer, total_llm_model_tokens, total_embedding_model_tokens
  
  def get_answer_from_llama_file(messages, file):
      dialog_messages = format_dialog_messages(messages)
      lang_code = get_language_code(remove_prompt_from_text(messages[-1]))
      index_name = get_index_name_from_file(file)
      index = get_index_from_file_cache(index_name)
      if index is None:
          logging.info(f"=====> Build index from file!")
          documents = SimpleDirectoryReader(input_files=[file]).load_data()
          index = GPTVectorStoreIndex.from_documents(documents, service_context=service_context)
          index.set_index_id(index_name)
          index.storage_context.persist()
          logging.info(
              f"=====> Save index to disk path: {index_cache_file_dir / index_name}")
      prompt = get_prompt_template(lang_code)
      logging.info('=====> Use llama file with chatGPT to answer!')
      logging.info('=====> dialog_messages')
      logging.info(dialog_messages)
      logging.info('=====> text_qa_template')
      logging.info(prompt)
      answer = answer = index.as_query_engine(text_qa_template=prompt).query(dialog_messages)
      total_llm_model_tokens = llm_predictor.last_token_usage
      total_embedding_model_tokens = service_context.embed_model.last_token_usage
      return answer, total_llm_model_tokens, total_embedding_model_tokens
  
  def get_text_from_whisper(voice_file_path):
      with open(voice_file_path, "rb") as f:
          transcript = openai.Audio.transcribe("whisper-1", f)
      return transcript.text
  
  lang_code_voice_map = {
      'zh': ['zh-CN-XiaoxiaoNeural', 'zh-CN-XiaohanNeural', 'zh-CN-YunxiNeural', 'zh-CN-YunyangNeural'],
      'en': ['en-US-JennyNeural', 'en-US-RogerNeural', 'en-IN-NeerjaNeural', 'en-IN-PrabhatNeural', 'en-AU-AnnetteNeural', 'en-AU-CarlyNeural', 'en-GB-AbbiNeural', 'en-GB-AlfieNeural'],
      'ja': ['ja-JP-AoiNeural', 'ja-JP-DaichiNeural'],
      'de': ['de-DE-AmalaNeural', 'de-DE-BerndNeural'],
  }
  
  def convert_to_ssml(text, voice_name=None):
      try:
          logging.info("=====> Convert text to ssml!")
          logging.info(text)
          text = remove_prompt_from_text(text)
          lang_code = get_language_code(text)
          if voice_name is None:
              voice_name = random.choice(lang_code_voice_map[lang_code])
      except Exception as e:
          logging.warning(f"Error: {e}. Using default voice.")
          voice_name = random.choice(lang_code_voice_map['zh'])
      ssml = '<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="zh-CN">'
      ssml += f'<voice name="{voice_name}">{text}</voice>'
      ssml += '</speak>'
  
      return ssml
  
  def get_voice_file_from_text(text, voice_name=None):
      speech_config = SpeechConfig(subscription=SPEECH_KEY, region=SPEECH_REGION)
      speech_config.set_speech_synthesis_output_format(
          SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3)
      speech_config.speech_synthesis_language = "zh-CN"
      file_name = f"{index_cache_voice_dir}{uuid.uuid4()}.mp3"
      file_config = AudioOutputConfig(filename=file_name)
      synthesizer = SpeechSynthesizer(
          speech_config=speech_config, audio_config=file_config)
      ssml = convert_to_ssml(text, voice_name)
      result = synthesizer.speak_ssml_async(ssml).get()
      if result.reason == ResultReason.SynthesizingAudioCompleted:
          logging.info("Speech synthesized for text [{}], and the audio was saved to [{}]".format(
              text, file_name))
      elif result.reason == ResultReason.Canceled:
          cancellation_details = result.cancellation_details
          logging.info("Speech synthesis canceled: {}".format(
              cancellation_details.reason))
          if cancellation_details.reason == CancellationReason.Error:
              logging.error("Error details: {}".format(
                  cancellation_details.error_details))
      return file_name
  
apis: llama_index.GPTVectorStoreIndex.from_documents llama_index.ServiceContext.from_defaults llama_index.readers.schema.base.Document llama_index.StorageContext.from_defaults llama_index.SimpleDirectoryReader llama_index.RssReader llama_index.load_index_from_storage
extract_api: [((795, 827), 'os.environ.get', 'os.environ.get', (['"""OPENAI_API_KEY"""'], {}), "('OPENAI_API_KEY')\n", (809, 827), False, 'import os\n'), ((841, 869), 'os.environ.get', 'os.environ.get', (['"""SPEECH_KEY"""'], {}), "('SPEECH_KEY')\n", (855, 869), False, 'import os\n'), ((886, 917), 'os.environ.get', 'os.environ.get', (['"""SPEECH_REGION"""'], {}), "('SPEECH_REGION')\n", (900, 917), False, 'import os\n'), ((973, 1008), 'pathlib.Path', 'Path', (['"""/tmp/myGPTReader/cache_web/"""'], {}), "('/tmp/myGPTReader/cache_web/')\n", (977, 1008), False, 'from pathlib import Path\n'), ((1032, 1063), 'pathlib.Path', 'Path', (['"""/data/myGPTReader/file/"""'], {}), "('/data/myGPTReader/file/')\n", (1036, 1063), False, 'from pathlib import Path\n'), ((1088, 1119), 'pathlib.Path', 'Path', (['"""/tmp/myGPTReader/voice/"""'], {}), "('/tmp/myGPTReader/voice/')\n", (1092, 1119), False, 'from pathlib import Path\n'), ((1530, 1587), 'llama_index.ServiceContext.from_defaults', 'ServiceContext.from_defaults', ([], {'llm_predictor': 'llm_predictor'}), '(llm_predictor=llm_predictor)\n', (1558, 1587), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((1611, 1641), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {}), '()\n', (1639, 1641), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((1665, 1695), 'llama_index.StorageContext.from_defaults', 'StorageContext.from_defaults', ([], {}), '()\n', (1693, 1695), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((2026, 2058), 'app.fetch_web_post.get_youtube_transcript', 'get_youtube_transcript', (['video_id'], {}), '(video_id)\n', (2048, 2058), False, 'from app.fetch_web_post import get_urls, get_youtube_transcript, scrape_website, scrape_website_by_phantomjscloud\n'), ((2117, 2137), 'llama_index.readers.schema.base.Document', 'Document', (['transcript'], {}), '(transcript)\n', (2125, 2137), False, 'from llama_index.readers.schema.base import Document\n'), ((3870, 3915), 'logging.info', 'logging.info', (['"""=====> Use chatGPT to answer!"""'], {}), "('=====> Use chatGPT to answer!')\n", (3882, 3915), False, 'import logging\n'), ((3920, 3949), 'logging.info', 'logging.info', (['dialog_messages'], {}), '(dialog_messages)\n', (3932, 3949), False, 'import logging\n'), ((3967, 4079), 'openai.ChatCompletion.create', 'openai.ChatCompletion.create', ([], {'model': '"""gpt-3.5-turbo"""', 'messages': "[{'role': 'user', 'content': dialog_messages}]"}), "(model='gpt-3.5-turbo', messages=[{'role':\n    'user', 'content': dialog_messages}])\n", (3995, 4079), False, 'import openai\n'), ((4102, 4132), 'logging.info', 'logging.info', (['completion.usage'], {}), '(completion.usage)\n', (4114, 4132), False, 'import logging\n'), ((4448, 4462), 'app.fetch_web_post.get_urls', 'get_urls', (['urls'], {}), '(urls)\n', (4456, 4462), False, 'from app.fetch_web_post import get_urls, get_youtube_transcript, scrape_website, scrape_website_by_phantomjscloud\n'), ((4467, 4495), 'logging.info', 'logging.info', (['combained_urls'], {}), '(combained_urls)\n', (4479, 4495), False, 'import logging\n'), ((5063, 5093), 'app.prompt.get_prompt_template', 'get_prompt_template', (['lang_code'], {}), '(lang_code)\n', (5082, 5093), False, 'from app.prompt import get_prompt_template\n'), ((5098, 5158), 'logging.info', 'logging.info', (['"""=====> Use llama web with chatGPT to answer!"""'], {}), "('=====> Use llama web with chatGPT to answer!')\n", (5110, 5158), False, 'import logging\n'), ((5163, 5201), 'logging.info', 'logging.info', (['"""=====> dialog_messages"""'], {}), "('=====> dialog_messages')\n", (5175, 5201), False, 'import logging\n'), ((5206, 5235), 'logging.info', 'logging.info', (['dialog_messages'], {}), '(dialog_messages)\n', (5218, 5235), False, 'import logging\n'), ((5240, 5279), 'logging.info', 'logging.info', (['"""=====> text_qa_template"""'], {}), "('=====> text_qa_template')\n", (5252, 5279), False, 'import logging\n'), ((5284, 5311), 'logging.info', 'logging.info', (['prompt.prompt'], {}), '(prompt.prompt)\n', (5296, 5311), False, 'import logging\n'), ((6326, 6356), 'app.prompt.get_prompt_template', 'get_prompt_template', (['lang_code'], {}), '(lang_code)\n', (6345, 6356), False, 'from app.prompt import get_prompt_template\n'), ((6361, 6422), 'logging.info', 'logging.info', (['"""=====> Use llama file with chatGPT to answer!"""'], {}), "('=====> Use llama file with chatGPT to answer!')\n", (6373, 6422), False, 'import logging\n'), ((6427, 6465), 'logging.info', 'logging.info', (['"""=====> dialog_messages"""'], {}), "('=====> dialog_messages')\n", (6439, 6465), False, 'import logging\n'), ((6470, 6499), 'logging.info', 'logging.info', (['dialog_messages'], {}), '(dialog_messages)\n', (6482, 6499), False, 'import logging\n'), ((6504, 6543), 'logging.info', 'logging.info', (['"""=====> text_qa_template"""'], {}), "('=====> text_qa_template')\n", (6516, 6543), False, 'import logging\n'), ((6548, 6568), 'logging.info', 'logging.info', (['prompt'], {}), '(prompt)\n', (6560, 6568), False, 'import logging\n'), ((8212, 8271), 'azure.cognitiveservices.speech.SpeechConfig', 'SpeechConfig', ([], {'subscription': 'SPEECH_KEY', 'region': 'SPEECH_REGION'}), '(subscription=SPEECH_KEY, region=SPEECH_REGION)\n', (8224, 8271), False, 'from azure.cognitiveservices.speech import SpeechConfig, SpeechSynthesizer, ResultReason, CancellationReason, SpeechSynthesisOutputFormat\n'), ((8524, 8561), 'azure.cognitiveservices.speech.audio.AudioOutputConfig', 'AudioOutputConfig', ([], {'filename': 'file_name'}), '(filename=file_name)\n', (8541, 8561), False, 'from azure.cognitiveservices.speech.audio import AudioOutputConfig\n'), ((8580, 8652), 'azure.cognitiveservices.speech.SpeechSynthesizer', 'SpeechSynthesizer', ([], {'speech_config': 'speech_config', 'audio_config': 'file_config'}), '(speech_config=speech_config, audio_config=file_config)\n', (8597, 8652), False, 'from azure.cognitiveservices.speech import SpeechConfig, SpeechSynthesizer, ResultReason, CancellationReason, SpeechSynthesisOutputFormat\n'), ((1451, 1504), 'langchain.chat_models.ChatOpenAI', 'ChatOpenAI', ([], {'temperature': '(0)', 'model_name': '"""gpt-3.5-turbo"""'}), "(temperature=0, model_name='gpt-3.5-turbo')\n", (1461, 1504), False, 'from langchain.chat_models import ChatOpenAI\n'), ((3209, 3268), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['web_storage_context'], {'index_id': 'name'}), '(web_storage_context, index_id=name)\n', (3232, 3268), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((3421, 3481), 'llama_index.load_index_from_storage', 'load_index_from_storage', (['file_storage_context'], {'index_id': 'name'}), '(file_storage_context, index_id=name)\n', (3444, 3481), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((4623, 4668), 'logging.info', 'logging.info', (['f"""=====> Build index from web!"""'], {}), "(f'=====> Build index from web!')\n", (4635, 4668), False, 'import logging\n'), ((4737, 4760), 'logging.info', 'logging.info', (['documents'], {}), '(documents)\n', (4749, 4760), False, 'import logging\n'), ((4777, 4855), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['documents'], {'service_context': 'service_context'}), '(documents, service_context=service_context)\n', (4811, 4855), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((4948, 5041), 'logging.info', 'logging.info', (['f"""=====> Save index to disk path: {index_cache_web_dir / index_file_name}"""'], {}), "(\n    f'=====> Save index to disk path: {index_cache_web_dir / index_file_name}')\n", (4960, 5041), False, 'import logging\n'), ((5912, 5958), 'logging.info', 'logging.info', (['f"""=====> Build index from file!"""'], {}), "(f'=====> Build index from file!')\n", (5924, 5958), False, 'import logging\n'), ((6049, 6127), 'llama_index.GPTVectorStoreIndex.from_documents', 'GPTVectorStoreIndex.from_documents', (['documents'], {'service_context': 'service_context'}), '(documents, service_context=service_context)\n', (6083, 6127), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((6215, 6304), 'logging.info', 'logging.info', (['f"""=====> Save index to disk path: {index_cache_file_dir / index_name}"""'], {}), "(\n    f'=====> Save index to disk path: {index_cache_file_dir / index_name}')\n", (6227, 6304), False, 'import logging\n'), ((6982, 7021), 'openai.Audio.transcribe', 'openai.Audio.transcribe', (['"""whisper-1"""', 'f'], {}), "('whisper-1', f)\n", (7005, 7021), False, 'import openai\n'), ((7532, 7576), 'logging.info', 'logging.info', (['"""=====> Convert text to ssml!"""'], {}), "('=====> Convert text to ssml!')\n", (7544, 7576), False, 'import logging\n'), ((7585, 7603), 'logging.info', 'logging.info', (['text'], {}), '(text)\n', (7597, 7603), False, 'import logging\n'), ((7669, 7692), 'app.util.get_language_code', 'get_language_code', (['text'], {}), '(text)\n', (7686, 7692), False, 'from app.util import get_language_code, get_youtube_video_id\n'), ((2339, 2358), 'app.fetch_web_post.scrape_website', 'scrape_website', (['url'], {}), '(url)\n', (2353, 2358), False, 'from app.fetch_web_post import get_urls, get_youtube_transcript, scrape_website, scrape_website_by_phantomjscloud\n'), ((2844, 2869), 'app.util.get_youtube_video_id', 'get_youtube_video_id', (['url'], {}), '(url)\n', (2864, 2869), False, 'from app.util import get_language_code, get_youtube_video_id\n'), ((3304, 3320), 'logging.error', 'logging.error', (['e'], {}), '(e)\n', (3317, 3320), False, 'import logging\n'), ((3517, 3533), 'logging.error', 'logging.error', (['e'], {}), '(e)\n', (3530, 3533), False, 'import logging\n'), ((7749, 7794), 'random.choice', 'random.choice', (['lang_code_voice_map[lang_code]'], {}), '(lang_code_voice_map[lang_code])\n', (7762, 7794), False, 'import random\n'), ((7830, 7882), 'logging.warning', 'logging.warning', (['f"""Error: {e}. Using default voice."""'], {}), "(f'Error: {e}. Using default voice.')\n", (7845, 7882), False, 'import logging\n'), ((7904, 7944), 'random.choice', 'random.choice', (["lang_code_voice_map['zh']"], {}), "(lang_code_voice_map['zh'])\n", (7917, 7944), False, 'import random\n'), ((8487, 8499), 'uuid.uuid4', 'uuid.uuid4', ([], {}), '()\n', (8497, 8499), False, 'import uuid\n'), ((2453, 2464), 'llama_index.RssReader', 'RssReader', ([], {}), '()\n', (2462, 2464), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((2664, 2701), 'app.fetch_web_post.scrape_website_by_phantomjscloud', 'scrape_website_by_phantomjscloud', (['url'], {}), '(url)\n', (2696, 2701), False, 'from app.fetch_web_post import get_urls, get_youtube_transcript, scrape_website, scrape_website_by_phantomjscloud\n'), ((5979, 6020), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', ([], {'input_files': '[file]'}), '(input_files=[file])\n', (6000, 6020), False, 'from llama_index import ServiceContext, GPTVectorStoreIndex, LLMPredictor, RssReader, SimpleDirectoryReader, StorageContext, load_index_from_storage\n'), ((3065, 3124), 'llama_index.readers.schema.base.Document', 'Document', (['f"""Can\'t get transcript from youtube video: {url}"""'], {}), '(f"Can\'t get transcript from youtube video: {url}")\n', (3073, 3124), False, 'from llama_index.readers.schema.base import Document\n'), ((3647, 3657), 'pathlib.Path', 'Path', (['file'], {}), '(file)\n', (3651, 3657), False, 'from pathlib import Path\n')]