code: # coding: utf-8
  import os
  import gradio as gr
  import re
  import uuid
  from PIL import Image, ImageDraw, ImageOps, ImageFont
  import numpy as np
  import argparse
  import inspect
  from langchain.agents.initialize import initialize_agent
  from langchain.agents.tools import Tool
  from langchain.chains.conversation.memory import ConversationBufferMemory
  from gpt4tools.llm import LlamaLangChain
  from gpt4tools.tools import *
  
  
  GPT4TOOLS_PREFIX = """GPT4Tools can handle various text and visual tasks, such as answering questions and providing in-depth explanations and discussions. It generates human-like text and uses tools to indirectly understand images. When referring to images, GPT4Tools follows strict file name rules. To complete visual tasks, GPT4Tools uses tools and stays loyal to observation outputs. Users can provide new images to GPT4Tools with a description, but tools must be used for subsequent tasks.
  TOOLS:
  ------
  
  GPT4Tools has access to the following tools:"""
  
  
  GPT4TOOLS_FORMAT_INSTRUCTIONS = """To use a tool, please use the following format:
  
  ```
  Thought: Do I need to use a tool? Yes
  Action: the action to take, should be one of [{tool_names}]
  Action Input: the input to the action
  Observation: the result of the action
  ```
  
  When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format:
  
  ```
  Thought: Do I need to use a tool? No
  {ai_prefix}: [your response here]
  ```
  """
  
  GPT4TOOLS_SUFFIX = """Follow file name rules and do not fake non-existent file names. Remember to provide the image file name loyally from the last tool observation.
  
  Previous conversation:
  {chat_history}
  
  New input: {input}
  GPT4Tools needs to use tools to observe images, not directly imagine them. Thoughts and observations in the conversation are only visible to GPT4Tools. When answering human questions, repeat important information. Let's think step by step.
  {agent_scratchpad}"""
  
  
  os.makedirs('image', exist_ok=True)
  
  
  def cut_dialogue_history(history_memory, keep_last_n_paragraphs=1):
      if history_memory is None or len(history_memory) == 0:
          return history_memory
      paragraphs = history_memory.split('Human:')
      if len(paragraphs) <= keep_last_n_paragraphs:
          return history_memory
      return 'Human:' + 'Human:'.join(paragraphs[-1:])
  
  class ConversationBot:
      def __init__(self, load_dict, llm_kwargs):
          # load_dict = {'VisualQuestionAnswering':'cuda:0', 'ImageCaptioning':'cuda:1',...}
          print(f"Initializing GPT4Tools, load_dict={load_dict}")
          if 'ImageCaptioning' not in load_dict:
              raise ValueError("You have to load ImageCaptioning as a basic function for GPT4Tools")
  
          self.models = {}
          # Load Basic Foundation Models
          for class_name, device in load_dict.items():
              self.models[class_name] = globals()[class_name](device=device)
  
          # Load Template Foundation Models
          for class_name, module in globals().items():
              if getattr(module, 'template_model', False):
                  template_required_names = {k for k in inspect.signature(module.__init__).parameters.keys() if k!='self'}
                  loaded_names = set([type(e).__name__ for e in self.models.values()])
                  if template_required_names.issubset(loaded_names):
                      self.models[class_name] = globals()[class_name](
                          **{name: self.models[name] for name in template_required_names})
          
          print(f"All the Available Functions: {self.models}")
  
          self.tools = []
          for instance in self.models.values():
              for e in dir(instance):
                  if e.startswith('inference'):
                      func = getattr(instance, e)
                      self.tools.append(Tool(name=func.name, description=func.description, func=func))
          self.llm = LlamaLangChain(model_kwargs=llm_kwargs) 
          self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')
  
      def init_agent(self, lang):
          self.memory.clear() #clear previous history
          if lang=='English':
              PREFIX, FORMAT_INSTRUCTIONS, SUFFIX = GPT4TOOLS_PREFIX, GPT4TOOLS_FORMAT_INSTRUCTIONS, GPT4TOOLS_SUFFIX
              place = "Enter text and press enter, or upload an image"
              label_clear = "Clear"
          else:
              raise NotImplementedError(f'{lang} is not supported yet')
          self.agent = initialize_agent(
              self.tools,
              self.llm,
              agent="conversational-react-description",
              verbose=True,
              memory=self.memory,
              return_intermediate_steps=True,
              agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS,
                            'suffix': SUFFIX}, )
          return gr.update(visible = True), gr.update(visible = False), gr.update(placeholder=place), gr.update(value=label_clear)
  
      def run_text(self, text, state, temperature, top_p, max_new_tokens, keep_last_n_paragraphs):
          self.llm.set_llm_params(temperature=temperature,
                                  top_p=top_p,
                                  max_new_tokens=max_new_tokens)
          self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_paragraphs)
          res = self.agent({"input": text.strip()})
          res['output'] = res['output'].replace("\\", "/")
          response = re.sub('(image/[-\w]*.png)', lambda m: f'![](file={m.group(0)})*{m.group(0)}*', res['output'])
          state = state + [(text, response)]
          print(f"\nProcessed run_text, Input text: {text}\nCurrent state: {state}\n"
                f"Current Memory: {self.agent.memory.buffer}")
          image_filenames = re.findall('image/.*.png', str(self.agent.memory.buffer))
          image_filename = image_filenames[-1] if len(image_filenames) > 0 else ''
          return state, state, f'{image_filename} '
  
      def run_image(self, image, state, txt, lang='English'):
          if image is None:
              return state, state, txt
          image_filename = os.path.join('image', f"{str(uuid.uuid4())[:8]}.png")
          print("======>Auto Resize Image...")
          img = image
          width, height = img.size
          ratio = min(512 / width, 512 / height)
          width_new, height_new = (round(width * ratio), round(height * ratio))
          width_new = int(np.round(width_new / 64.0)) * 64
          height_new = int(np.round(height_new / 64.0)) * 64
          img = img.resize((width_new, height_new))
          img = img.convert('RGB')
          img.save(image_filename, "PNG")
          print(f"Resize image form {width}x{height} to {width_new}x{height_new}")
          description = self.models['ImageCaptioning'].inference(image_filename)
          if lang == 'English':
              Human_prompt = f'\nHuman: Provide an image named {image_filename}. The description is: {description}. Understand the image using tools.\n'
              AI_prompt = "Received."
          else:
              raise NotImplementedError(f'{lang} is not supported yet')
          self.agent.memory.buffer = self.agent.memory.buffer + Human_prompt + 'AI: ' + AI_prompt
          state = state + [(f"![](file={image_filename})*{image_filename}*", AI_prompt)]
          print(f"\nProcessed run_image, Input image: {image_filename}\nCurrent state: {state}\n"
                f"Current Memory: {self.agent.memory.buffer}")
          return state, state, f'{image_filename} {txt}'
  
  
  if __name__ == '__main__':
      parser = argparse.ArgumentParser()
      parser.add_argument('--base_model', type=str, required=True, help='folder path to the vicuna with tokenizer')
      parser.add_argument('--lora_model', type=str, required=True, help='folder path to the lora model')
      parser.add_argument('--load', type=str, default='ImageCaptioning_cuda:0,Text2Image_cuda:0')
      parser.add_argument('--llm_device', type=str, default='cpu', help='device to run the llm model')
      parser.add_argument('--temperature', type=float, default=0.1, help='temperature for the llm model')
      parser.add_argument('--max_new_tokens', type=int, default=512, help='max number of new tokens to generate')
      parser.add_argument('--top_p', type=float, default=0.75, help='top_p for the llm model')
      parser.add_argument('--top_k', type=int, default=40, help='top_k for the llm model')
      parser.add_argument('--num_beams', type=int, default=1, help='num_beams for the llm model')
      parser.add_argument('--keep_last_n_paragraphs', type=int, default=1, help='keep last n paragraphs in the memory')
      parser.add_argument('--cache-dir', type=str, default=None, help="cache path to save model")
      parser.add_argument('--server-name', type=str, default='0.0.0.0', help="gradio sever name")
      parser.add_argument('--server-port', type=int, default=8888, help="gradio server port")
      parser.add_argument('--share', action="store_true")
      args = parser.parse_args()
  
      load_dict = {e.split('_')[0].strip(): e.split('_')[1].strip() for e in args.load.split(',')}
      llm_kwargs = {'base_model': args.base_model,
                    'lora_model': args.lora_model,
                    'device': args.llm_device,
                    'temperature': args.temperature,
                    'max_new_tokens': args.max_new_tokens,
                    'top_p': args.top_p,
                    'top_k': args.top_k,
                    'num_beams': args.num_beams,
                    'cache_dir': args.cache_dir,}
      bot = ConversationBot(load_dict=load_dict, llm_kwargs=llm_kwargs)
  
      examples = [
          ['asserts/images/example-1.jpg','Make the image look like a cartoon.'],
          ['asserts/images/example-2.jpg','Segment the tie in the image.'],
          ['asserts/images/example-3.jpg','Generate a man watching a sea based on the pose of the woman.'],
          ['asserts/images/example-4.jpg','Tell me a story about this image.'],
      ]
  
      with gr.Blocks() as demo:
          with gr.Row():
              with gr.Column(scale=0.3):
                  with gr.Row():
                      image = gr.Image(type="pil", label="input image")
                  with gr.Row():
                      txt = gr.Textbox(lines=7, show_label=False, elem_id="textbox",
                                       placeholder="Enter text and press submit, or upload an image").style(container=False)
                  with gr.Row():
                      submit = gr.Button("Submit")
                  with gr.Row():
                      clear = gr.Button("Clear")
                  with gr.Row():
                      keep_last_n_paragraphs = gr.Slider(
                          minimum=0,
                          maximum=3,
                          value=args.keep_last_n_paragraphs,
                          step=1,
                          interactive=True,
                          label="Remember Last N Paragraphs")
                      max_new_token = gr.Slider(
                          minimum=128,
                          maximum=1024,
                          value=args.max_new_tokens,
                          step=64,
                          interactive=True,
                          label="Max New Tokens")
                      temperature = gr.Slider(
                          minimum=0.0,
                          maximum=1.0,
                          value=args.temperature,
                          step=0.1,
                          interactive=True,
                          label="Temperature")
                      top_p = gr.Slider(
                          minimum=0.0,
                          maximum=1.0,
                          value=args.top_p,
                          step=0.1,
                          interactive=True,
                          label="Top P")
              with gr.Column(scale=0.7):
                  chatbot = gr.Chatbot(elem_id="chatbot", label="🦙 GPT4Tools").style(height=690)
                  state = gr.State([])
      
              # TODO: support more language 
              bot.init_agent('English')
              txt.submit(bot.run_text, [txt, state], [chatbot, state])
              txt.submit(lambda: "", None, txt)
              # submit.click(bot.run_image, [image, state, txt], [chatbot, state, txt]).then(
              #     bot.run_text, [txt, state, temperature, top_p, max_new_token, keep_last_n_paragraphs], [chatbot, state]).then(
              #         lambda: "", None, txt).then(
              #         lambda: None, None, image)
              submit.click(bot.run_image, [image, state, txt], [chatbot, state, txt]).then(
                  bot.run_text, [txt, state, temperature, top_p, max_new_token, keep_last_n_paragraphs], [chatbot, state, txt]).then(
                      lambda: None, None, image)
              clear.click(bot.memory.clear)
              clear.click(lambda: [], None, chatbot)
              clear.click(lambda: [], None, state)
          with gr.Row():
              gr.Examples(
                  examples=examples,
                  inputs=[image, txt],
              )
          demo.launch(server_name=args.server_name, server_port=args.server_port, enable_queue=True, share=args.share)
apis: langchain.chains.conversation.memory.ConversationBufferMemory langchain.agents.initialize.initialize_agent langchain.agents.tools.Tool
extract_api: [((1924, 1959), 'os.makedirs', 'os.makedirs', (['"""image"""'], {'exist_ok': '(True)'}), "('image', exist_ok=True)\n", (1935, 1959), False, 'import os\n'), ((7453, 7478), 'argparse.ArgumentParser', 'argparse.ArgumentParser', ([], {}), '()\n', (7476, 7478), False, 'import argparse\n'), ((3840, 3879), 'gpt4tools.llm.LlamaLangChain', 'LlamaLangChain', ([], {'model_kwargs': 'llm_kwargs'}), '(model_kwargs=llm_kwargs)\n', (3854, 3879), False, 'from gpt4tools.llm import LlamaLangChain\n'), ((3903, 3975), 'langchain.chains.conversation.memory.ConversationBufferMemory', 'ConversationBufferMemory', ([], {'memory_key': '"""chat_history"""', 'output_key': '"""output"""'}), "(memory_key='chat_history', output_key='output')\n", (3927, 3975), False, 'from langchain.chains.conversation.memory import ConversationBufferMemory\n'), ((4413, 4667), 'langchain.agents.initialize.initialize_agent', 'initialize_agent', (['self.tools', 'self.llm'], {'agent': '"""conversational-react-description"""', 'verbose': '(True)', 'memory': 'self.memory', 'return_intermediate_steps': '(True)', 'agent_kwargs': "{'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS, 'suffix': SUFFIX\n    }"}), "(self.tools, self.llm, agent=\n    'conversational-react-description', verbose=True, memory=self.memory,\n    return_intermediate_steps=True, agent_kwargs={'prefix': PREFIX,\n    'format_instructions': FORMAT_INSTRUCTIONS, 'suffix': SUFFIX})\n", (4429, 4667), False, 'from langchain.agents.initialize import initialize_agent\n'), ((9840, 9851), 'gradio.Blocks', 'gr.Blocks', ([], {}), '()\n', (9849, 9851), True, 'import gradio as gr\n'), ((4783, 4806), 'gradio.update', 'gr.update', ([], {'visible': '(True)'}), '(visible=True)\n', (4792, 4806), True, 'import gradio as gr\n'), ((4810, 4834), 'gradio.update', 'gr.update', ([], {'visible': '(False)'}), '(visible=False)\n', (4819, 4834), True, 'import gradio as gr\n'), ((4838, 4866), 'gradio.update', 'gr.update', ([], {'placeholder': 'place'}), '(placeholder=place)\n', (4847, 4866), True, 'import gradio as gr\n'), ((4868, 4896), 'gradio.update', 'gr.update', ([], {'value': 'label_clear'}), '(value=label_clear)\n', (4877, 4896), True, 'import gradio as gr\n'), ((9874, 9882), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (9880, 9882), True, 'import gradio as gr\n'), ((12740, 12748), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (12746, 12748), True, 'import gradio as gr\n'), ((12762, 12813), 'gradio.Examples', 'gr.Examples', ([], {'examples': 'examples', 'inputs': '[image, txt]'}), '(examples=examples, inputs=[image, txt])\n', (12773, 12813), True, 'import gradio as gr\n'), ((6340, 6366), 'numpy.round', 'np.round', (['(width_new / 64.0)'], {}), '(width_new / 64.0)\n', (6348, 6366), True, 'import numpy as np\n'), ((6398, 6425), 'numpy.round', 'np.round', (['(height_new / 64.0)'], {}), '(height_new / 64.0)\n', (6406, 6425), True, 'import numpy as np\n'), ((9901, 9921), 'gradio.Column', 'gr.Column', ([], {'scale': '(0.3)'}), '(scale=0.3)\n', (9910, 9921), True, 'import gradio as gr\n'), ((11640, 11660), 'gradio.Column', 'gr.Column', ([], {'scale': '(0.7)'}), '(scale=0.7)\n', (11649, 11660), True, 'import gradio as gr\n'), ((11781, 11793), 'gradio.State', 'gr.State', (['[]'], {}), '([])\n', (11789, 11793), True, 'import gradio as gr\n'), ((9944, 9952), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (9950, 9952), True, 'import gradio as gr\n'), ((9982, 10023), 'gradio.Image', 'gr.Image', ([], {'type': '"""pil"""', 'label': '"""input image"""'}), "(type='pil', label='input image')\n", (9990, 10023), True, 'import gradio as gr\n'), ((10045, 10053), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (10051, 10053), True, 'import gradio as gr\n'), ((10282, 10290), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (10288, 10290), True, 'import gradio as gr\n'), ((10321, 10340), 'gradio.Button', 'gr.Button', (['"""Submit"""'], {}), "('Submit')\n", (10330, 10340), True, 'import gradio as gr\n'), ((10362, 10370), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (10368, 10370), True, 'import gradio as gr\n'), ((10400, 10418), 'gradio.Button', 'gr.Button', (['"""Clear"""'], {}), "('Clear')\n", (10409, 10418), True, 'import gradio as gr\n'), ((10440, 10448), 'gradio.Row', 'gr.Row', ([], {}), '()\n', (10446, 10448), True, 'import gradio as gr\n'), ((10495, 10627), 'gradio.Slider', 'gr.Slider', ([], {'minimum': '(0)', 'maximum': '(3)', 'value': 'args.keep_last_n_paragraphs', 'step': '(1)', 'interactive': '(True)', 'label': '"""Remember Last N Paragraphs"""'}), "(minimum=0, maximum=3, value=args.keep_last_n_paragraphs, step=1,\n    interactive=True, label='Remember Last N Paragraphs')\n", (10504, 10627), True, 'import gradio as gr\n'), ((10805, 10923), 'gradio.Slider', 'gr.Slider', ([], {'minimum': '(128)', 'maximum': '(1024)', 'value': 'args.max_new_tokens', 'step': '(64)', 'interactive': '(True)', 'label': '"""Max New Tokens"""'}), "(minimum=128, maximum=1024, value=args.max_new_tokens, step=64,\n    interactive=True, label='Max New Tokens')\n", (10814, 10923), True, 'import gradio as gr\n'), ((11099, 11211), 'gradio.Slider', 'gr.Slider', ([], {'minimum': '(0.0)', 'maximum': '(1.0)', 'value': 'args.temperature', 'step': '(0.1)', 'interactive': '(True)', 'label': '"""Temperature"""'}), "(minimum=0.0, maximum=1.0, value=args.temperature, step=0.1,\n    interactive=True, label='Temperature')\n", (11108, 11211), True, 'import gradio as gr\n'), ((11381, 11482), 'gradio.Slider', 'gr.Slider', ([], {'minimum': '(0.0)', 'maximum': '(1.0)', 'value': 'args.top_p', 'step': '(0.1)', 'interactive': '(True)', 'label': '"""Top P"""'}), "(minimum=0.0, maximum=1.0, value=args.top_p, step=0.1, interactive\n    =True, label='Top P')\n", (11390, 11482), True, 'import gradio as gr\n'), ((3758, 3819), 'langchain.agents.tools.Tool', 'Tool', ([], {'name': 'func.name', 'description': 'func.description', 'func': 'func'}), '(name=func.name, description=func.description, func=func)\n', (3762, 3819), False, 'from langchain.agents.tools import Tool\n'), ((11688, 11738), 'gradio.Chatbot', 'gr.Chatbot', ([], {'elem_id': '"""chatbot"""', 'label': '"""🦙 GPT4Tools"""'}), "(elem_id='chatbot', label='🦙 GPT4Tools')\n", (11698, 11738), True, 'import gradio as gr\n'), ((6068, 6080), 'uuid.uuid4', 'uuid.uuid4', ([], {}), '()\n', (6078, 6080), False, 'import uuid\n'), ((10081, 10205), 'gradio.Textbox', 'gr.Textbox', ([], {'lines': '(7)', 'show_label': '(False)', 'elem_id': '"""textbox"""', 'placeholder': '"""Enter text and press submit, or upload an image"""'}), "(lines=7, show_label=False, elem_id='textbox', placeholder=\n    'Enter text and press submit, or upload an image')\n", (10091, 10205), True, 'import gradio as gr\n'), ((3072, 3106), 'inspect.signature', 'inspect.signature', (['module.__init__'], {}), '(module.__init__)\n', (3089, 3106), False, 'import inspect\n')]