code: import argparse
  import copy
  import logging
  import os
  import sys
  import warnings
  from typing import Optional, List, Callable
  from langchain.llms import OpenAI
  import faiss
  import gradio as gr
  import torch
  import torch.distributed as dist
  import transformers
  from accelerate import dispatch_model, infer_auto_device_map
  from accelerate.hooks import (
      AlignDevicesHook,
      add_hook_to_module,
      remove_hook_from_submodules,
  )
  from accelerate.utils import get_balanced_memory
  from huggingface_hub import hf_hub_download
  from llama_index import LLMPredictor
  from llama_index import PromptHelper, SimpleDirectoryReader
  from llama_index import ServiceContext
  from llama_index import GPTKeywordTableIndex, GPTSimpleVectorIndex, GPTListIndex, GPTTreeIndex, GPTFaissIndex
  from peft import PeftModelForCausalLM, LoraConfig
  from peft.utils import PeftType, set_peft_model_state_dict
  from torch import nn
  from transformers.deepspeed import is_deepspeed_zero3_enabled
  from transformers.generation.beam_search import BeamSearchScorer
  from transformers.generation.utils import (
      LogitsProcessorList,
      StoppingCriteriaList,
      GenerationMixin,
  )
  
  from model import CustomLLM, Llama7bHFLLM
  
  assert (
          "LlamaTokenizer" in transformers._import_structure["models.llama"]
  ), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"
  from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig
  
  
  class SteamGenerationMixin(PeftModelForCausalLM, GenerationMixin):
      # support for streamly beam search
      @torch.no_grad()
      def stream_generate(
          self,
          input_ids: Optional[torch.Tensor] = None,
          generation_config: Optional[GenerationConfig] = None,
          logits_processor: Optional[LogitsProcessorList] = None,
          stopping_criteria: Optional[StoppingCriteriaList] = None,
          prefix_allowed_tokens_fn: Optional[
              Callable[[int, torch.Tensor], List[int]]
          ] = None,
          **kwargs,
      ):
          self._reorder_cache = self.base_model._reorder_cache
          if is_deepspeed_zero3_enabled() and dist.world_size() > 1:
              synced_gpus = True
          else:
              synced_gpus = False
  
          if kwargs.get("attention_mask", None) is not None:
              # concat prompt attention mask
              prefix_attention_mask = torch.ones(
                  kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens
              ).to(kwargs["input_ids"].device)
              kwargs["attention_mask"] = torch.cat(
                  (prefix_attention_mask, kwargs["attention_mask"]), dim=1
              )
          if kwargs.get("position_ids", None) is not None:
              warnings.warn(
                  "Position ids are not supported for parameter efficient tuning. Ignoring position ids."
              )
              kwargs["position_ids"] = None
          if kwargs.get("token_type_ids", None) is not None:
              warnings.warn(
                  "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"
              )
              kwargs["token_type_ids"] = None
  
          batch_size, input_ids_seq_length = input_ids.shape[0], input_ids.shape[-1]
  
          if generation_config is None:
              generation_config = self.generation_config
          generation_config = copy.deepcopy(generation_config)
          model_kwargs = generation_config.update(**kwargs)
  
          bos_token_id, eos_token_id, pad_token_id = (
              generation_config.bos_token_id,
              generation_config.eos_token_id,
              generation_config.pad_token_id,
          )
  
          if isinstance(eos_token_id, int):
              eos_token_id = [eos_token_id]
  
          has_default_max_length = (
              kwargs.get("max_length") is None
              and generation_config.max_length is not None
          )
          if has_default_max_length and generation_config.max_new_tokens is None:
              warnings.warn(
                  f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
                  "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
                  " recommend using `max_new_tokens` to control the maximum length of the generation.",
                  UserWarning,
              )
          elif generation_config.max_new_tokens is not None:
              generation_config.max_length = (
                  generation_config.max_new_tokens + input_ids_seq_length
              )
          if generation_config.min_new_tokens is not None:
              generation_config.min_length = (
                  generation_config.min_new_tokens + input_ids_seq_length
              )
  
          if input_ids_seq_length >= generation_config.max_length:
              input_ids_string = (
                  "decoder_input_ids" if self.config.is_encoder_decoder else "input_ids"
              )
  
          # 2. Set generation parameters if not already defined
          logits_processor = (
              logits_processor if logits_processor is not None else LogitsProcessorList()
          )
          stopping_criteria = (
              stopping_criteria
              if stopping_criteria is not None
              else StoppingCriteriaList()
          )
          # 8. prepare distribution pre_processing samplers
          logits_processor = self._get_logits_processor(
              generation_config=generation_config,
              input_ids_seq_length=input_ids_seq_length,
              encoder_input_ids=input_ids,
              prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
              logits_processor=logits_processor,
          )
          # 9. prepare stopping criteria
          stopping_criteria = self._get_stopping_criteria(
              generation_config=generation_config, stopping_criteria=stopping_criteria
          )
          logits_warper = self._get_logits_warper(generation_config)
  
          # 10. go into beam search generation modes
          # 11. prepare beam search scorer
          num_beams = generation_config.num_beams
          beam_scorer = BeamSearchScorer(
              batch_size=batch_size,
              num_beams=generation_config.num_beams,
              device=input_ids.device,
              length_penalty=generation_config.length_penalty,
              do_early_stopping=generation_config.early_stopping,
              num_beam_hyps_to_keep=generation_config.num_return_sequences,
              max_length=generation_config.max_length,
          )
          # 12. interleave input_ids with `num_beams` additional sequences per batch
          input_ids, model_kwargs = self._expand_inputs_for_generation(
              input_ids=input_ids,
              expand_size=generation_config.num_beams,
              is_encoder_decoder=self.config.is_encoder_decoder,
              **model_kwargs,
          )
  
          # beam_search logits
          batch_beam_size, cur_len = input_ids.shape
          if num_beams * batch_size != batch_beam_size:
              raise ValueError(
                  f"Batch dimension of `input_ids` should be {num_beams * batch_size}, but is {batch_beam_size}."
              )
          beam_scores = torch.zeros(
              (batch_size, num_beams), dtype=torch.float, device=input_ids.device
          )
          beam_scores[:, 1:] = -1e9
          beam_scores = beam_scores.view((batch_size * num_beams,))
          this_peer_finished = False  # used by synced_gpus only
          while True:
              if synced_gpus:
                  # Under synced_gpus the `forward` call must continue until all gpus complete their sequence.
                  # The following logic allows an early break if all peers finished generating their sequence
                  this_peer_finished_flag = torch.tensor(
                      0.0 if this_peer_finished else 1.0
                  ).to(input_ids.device)
                  # send 0.0 if we finished, 1.0 otherwise
                  dist.all_reduce(this_peer_finished_flag, op=dist.ReduceOp.SUM)
                  # did all peers finish? the reduced sum will be 0.0 then
                  if this_peer_finished_flag.item() == 0.0:
                      break
  
              model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
              outputs = self(
                  **model_inputs,
                  return_dict=True,
                  output_attentions=False,
                  output_hidden_states=False,
              )
  
              if synced_gpus and this_peer_finished:
                  cur_len = cur_len + 1
                  continue  # don't waste resources running the code we don't need
  
              next_token_logits = outputs.logits[:, -1, :]
              # next_token_logits = self.adjust_logits_during_generation(next_token_logits, cur_len=cur_len) hack: adjust tokens for Marian.
              next_token_scores = nn.functional.log_softmax(
                  next_token_logits, dim=-1
              )  # (batch_size * num_beams, vocab_size)
              next_token_scores_processed = logits_processor(input_ids, next_token_scores)
              next_token_scores = next_token_scores_processed + beam_scores[
                  :, None
              ].expand_as(next_token_scores)
  
              # reshape for beam search
              vocab_size = next_token_scores.shape[-1]
              next_token_scores = next_token_scores.view(
                  batch_size, num_beams * vocab_size
              )
  
              # Sample 2 next tokens for each beam (so we have some spare tokens and match output of beam search)
              next_token_scores, next_tokens = torch.topk(
                  next_token_scores, 2 * num_beams, dim=1, largest=True, sorted=True
              )
              next_indices = torch.div(next_tokens, vocab_size, rounding_mode="floor")
              next_tokens = next_tokens % vocab_size
              # stateless
              beam_outputs = beam_scorer.process(
                  input_ids,
                  next_token_scores,
                  next_tokens,
                  next_indices,
                  pad_token_id=pad_token_id,
                  eos_token_id=eos_token_id,
                  beam_indices=None,
              )
              beam_scores = beam_outputs["next_beam_scores"]
              beam_next_tokens = beam_outputs["next_beam_tokens"]
              beam_idx = beam_outputs["next_beam_indices"]
  
              input_ids = torch.cat(
                  [input_ids[beam_idx, :], beam_next_tokens.unsqueeze(-1)], dim=-1
              )
              model_kwargs = self._update_model_kwargs_for_generation(
                  outputs, model_kwargs, is_encoder_decoder=self.config.is_encoder_decoder
              )
              if model_kwargs["past_key_values"] is not None:
                  model_kwargs["past_key_values"] = self._reorder_cache(
                      model_kwargs["past_key_values"], beam_idx
                  )
  
              # increase cur_len
              cur_len = cur_len + 1
  
              yield input_ids
  
              if beam_scorer.is_done or stopping_criteria(input_ids, None):
                  if not synced_gpus:
                      break
                  else:
                      this_peer_finished = True
  
          final_result = beam_scorer.finalize(
              input_ids,
              beam_scores,
              next_tokens,
              next_indices,
              pad_token_id=pad_token_id,
              eos_token_id=eos_token_id,
              max_length=stopping_criteria.max_length,
              beam_indices=None,
          )
          yield final_result["sequences"]
  
      # default it call `model = MODEL_TYPE_TO_PEFT_MODEL_MAPPING[config.task_type](model, config)`, not cls!! so inherent PeftModelForCausalLM is no sense
      @classmethod
      def from_pretrained(cls, model, model_id, **kwargs):
          # load the config
          config = LoraConfig.from_pretrained(model_id)
  
          if getattr(model, "hf_device_map", None) is not None:
              remove_hook_from_submodules(model)
  
          # here is the hack
          model = cls(model, config)
  
          # load weights if any
          if os.path.exists(os.path.join(model_id, "adapter_model.bin")):
              filename = os.path.join(model_id, "adapter_model.bin")
          else:
              try:
                  filename = hf_hub_download(model_id, "adapter_model.bin")
              except:  # noqa
                  raise ValueError(
                      f"Can't find weights for {model_id} in {model_id} or in the Hugging Face Hub. "
                      f"Please check that the file {'adapter_model.bin'} is present at {model_id}."
                  )
  
          adapters_weights = torch.load(
              filename,
              map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu"),
          )
          # load the weights into the model
          model = set_peft_model_state_dict(model, adapters_weights)
          if getattr(model, "hf_device_map", None) is not None:
              device_map = kwargs.get("device_map", "auto")
              max_memory = kwargs.get("max_memory", None)
              no_split_module_classes = model._no_split_modules
              if device_map != "sequential":
                  max_memory = get_balanced_memory(
                      model,
                      max_memory=max_memory,
                      no_split_module_classes=no_split_module_classes,
                      low_zero=(device_map == "balanced_low_0"),
                  )
              if isinstance(device_map, str):
                  device_map = infer_auto_device_map(
                      model,
                      max_memory=max_memory,
                      no_split_module_classes=no_split_module_classes,
                  )
              model = dispatch_model(model, device_map=device_map)
              hook = AlignDevicesHook(io_same_device=True)
              if model.peft_config.peft_type == PeftType.LORA:
                  add_hook_to_module(model.base_model.model, hook)
              else:
                  remove_hook_from_submodules(model.prompt_encoder)
                  add_hook_to_module(model.base_model, hook)
          return model
  
  
  parser = argparse.ArgumentParser()
  parser.add_argument("--model_path", type=str, default="decapoda-research/llama-7b-hf")
  parser.add_argument("--lora_path", type=str, default="./lora-Vicuna/checkpoint-3000")
  parser.add_argument("--use_local", type=int, default=1)
  args = parser.parse_args()
  
  tokenizer = LlamaTokenizer.from_pretrained(args.model_path)
  
  LOAD_8BIT = True
  BASE_MODEL = args.model_path
  LORA_WEIGHTS = args.lora_path
  
  
  
  # fix the path for local checkpoint
  lora_bin_path = os.path.join(args.lora_path, "adapter_model.bin")
  print(lora_bin_path)
  if not os.path.exists(lora_bin_path) and args.use_local:
      pytorch_bin_path = os.path.join(args.lora_path, "pytorch_model.bin")
      print(pytorch_bin_path)
      if os.path.exists(pytorch_bin_path):
          os.rename(pytorch_bin_path, lora_bin_path)
          warnings.warn(
              "The file name of the lora checkpoint'pytorch_model.bin' is replaced with 'adapter_model.bin'"
          )
      else:
          assert ('Checkpoint is not Found!')
  
  if torch.cuda.is_available():
      device = "cuda"
  else:
      device = "cpu"
  
  try:
      if torch.backends.mps.is_available():
          device = "mps"
  except:
      pass
  
  if device == "cuda":
      model = LlamaForCausalLM.from_pretrained(
          BASE_MODEL,
          load_in_8bit=LOAD_8BIT,
          torch_dtype=torch.float16,
          device_map={"": 0},
      )
      model = SteamGenerationMixin.from_pretrained(
          model, LORA_WEIGHTS, torch_dtype=torch.float16, device_map={"": 0}
      )
  elif device == "mps":
      model = LlamaForCausalLM.from_pretrained(
          BASE_MODEL,
          device_map={"": device},
          torch_dtype=torch.float16,
      )
      model = SteamGenerationMixin.from_pretrained(
          model,
          LORA_WEIGHTS,
          device_map={"": device},
          torch_dtype=torch.float16,
      )
  else:
      model = LlamaForCausalLM.from_pretrained(
          BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True
      )
      model = SteamGenerationMixin.from_pretrained(
          model,
          LORA_WEIGHTS,
          device_map={"": device},
      )
  
  
  def generate_prompt(instruction, input=None):
      if input:
          return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
  
  ### Instruction:
  {instruction}
  
  ### Input:
  {input}
  
  ### Response:"""
      else:
          return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
  
  ### Instruction:
  {instruction}
  
  ### Response:"""
  
  
  if not LOAD_8BIT:
      model.half()  # seems to fix bugs for some users.
  
  model.eval()
  if torch.__version__ >= "2" and sys.platform != "win32":
      model = torch.compile(model)
  
  
  
  logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
  logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
  
  import openai
  
  openai.api_key = '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'
  os.environ["OPENAI_API_KEY"] = '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'
  
  
  
  def evaluate(
          input,
          temperature=0.1,
          top_p=0.75,
          top_k=40,
          num_beams=4,
          max_new_tokens=2500,
          min_new_tokens=1,
          repetition_penalty=2.0,
          **kwargs,
  ):
      print('start text llama-index')
      # TEST
      #
      # set maximum input size
      max_input_size = 2048
      # set number of output tokens
      num_output = 1024
      # set maximum chunk overlap
      max_chunk_overlap = 20
  
      gen_config = GenerationConfig(
          temperature=temperature,
          top_p=top_p,
          top_k=top_k,
          num_beams=num_beams,
          bos_token_id=1,
          eos_token_id=2,
          pad_token_id=0,
          max_new_tokens=max_new_tokens,
          # max_length=max_new_tokens+input_sequence
          min_new_tokens=min_new_tokens,
          # min_length=min_new_tokens+input_sequence
          repetition_penalty=repetition_penalty
  
      )
  
      # service_context = ServiceContext.from_defaults(
      #     llm_predictor=LLMPredictor(llm=CustomLLM(mod=model, token=tokenizer, gen_config=gen_config, device=device)),
      #     prompt_helper=PromptHelper(max_input_size, num_output, max_chunk_overlap))
  
      service_context = ServiceContext.from_defaults(
          llm_predictor=LLMPredictor(llm=model),
          prompt_helper=PromptHelper(max_input_size, num_output, max_chunk_overlap))
  
      documents = SimpleDirectoryReader('Chinese-Vicuna/index-docs').load_data()
      print(documents)
  
      print('start init index')
      # llm_predictor = LLMPredictor(llm=OpenAI(temperature=0, model_name="text-davinci-003"))
      # default_service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)
      # index = GPTFaissIndex.from_documents(documents, service_context=service_context)
      # index = GPTFaissIndex.from_documents(documents, faiss_index=faiss.IndexFlatL2(1536), service_context=default_service_context)
      print('end init index done')
      print('start save to disk')
      # index.save_to_disk("clash-index.json")
  
      # suffix do not matter
      faiss_index_save_path = 'faiss_index.faiss'
      faiss_index = faiss.IndexFlatL2(1536)
  
      faiss.write_index(faiss_index, faiss_index_save_path)
  
      index = GPTFaissIndex.load_from_disk(save_path='clash-index.json',
                                           faiss_index=faiss_index_save_path,
                                           service_context=service_context)
      print('end save to disk')
      # Query and print response
      print('start query')
      response = index.query(input)
      print('end query')
      print(response)
  
      return response
  
      # with torch.no_grad():
      #     # immOutPut = model.generate(input_ids=input_ids, generation_config=generation_config,
      #     #                            return_dict_in_generate=True, output_scores=False,
      #     #                            repetition_penalty=float(repetition_penalty), )
      #     # outputs = tokenizer.batch_decode(immOutPut)
      #     last_show_text = ''
      #     for generation_output in model.stream_generate(
      #             input_ids=input_ids,
      #             generation_config=generation_config,
      #             return_dict_in_generate=True,
      #             output_scores=False,
      #             repetition_penalty=float(repetition_penalty),
      #     ):
      #         outputs = tokenizer.batch_decode(generation_output)
      #         show_text = "\n--------------------------------------------\n".join(
      #             [output.split("### Response:")[1].strip().replace('�', '') for output in outputs]
      #         )
      #         # if show_text== '':
      #         #     yield last_show_text
      #         # else:
      #         yield show_text
      #         last_show_text = outputs[0].split("### Response:")[1].strip().replace('�', '')
  
  
  gr.Interface(
      fn=evaluate,
      inputs=[
          gr.components.Textbox(
              lines=2, label="Input", placeholder="Tell me about alpacas."
          ),
          gr.components.Slider(minimum=0, maximum=1, value=0.1, label="Temperature"),
          gr.components.Slider(minimum=0, maximum=1, value=0.75, label="Top p"),
          gr.components.Slider(minimum=0, maximum=100, step=1, value=40, label="Top k"),
          gr.components.Slider(minimum=1, maximum=10, step=1, value=4, label="Beams Number"),
          gr.components.Slider(
              minimum=1, maximum=2000, step=1, value=256, label="Max New Tokens"
          ),
          gr.components.Slider(
              minimum=1, maximum=100, step=1, value=1, label="Min New Tokens"
          ),
          gr.components.Slider(
              minimum=0.1, maximum=10.0, step=0.1, value=1.0, label="Repetition Penalty"
          ),
      ],
      outputs=[
          gr.inputs.Textbox(
              lines=15,
              label="Output",
          )
      ],
      title="Chinese-Vicuna 中文小羊驼",
      description="结合 llama-index prompt 搜索优化的 中文小羊驼",
  ).queue().launch(share=True)
  
apis: llama_index.GPTFaissIndex.load_from_disk llama_index.SimpleDirectoryReader llama_index.LLMPredictor llama_index.PromptHelper
extract_api: [((13996, 14021), 'argparse.ArgumentParser', 'argparse.ArgumentParser', ([], {}), '()\n', (14019, 14021), False, 'import argparse\n'), ((14291, 14338), 'transformers.LlamaTokenizer.from_pretrained', 'LlamaTokenizer.from_pretrained', (['args.model_path'], {}), '(args.model_path)\n', (14321, 14338), False, 'from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n'), ((14471, 14520), 'os.path.join', 'os.path.join', (['args.lora_path', '"""adapter_model.bin"""'], {}), "(args.lora_path, 'adapter_model.bin')\n", (14483, 14520), False, 'import os\n'), ((14990, 15015), 'torch.cuda.is_available', 'torch.cuda.is_available', ([], {}), '()\n', (15013, 15015), False, 'import torch\n'), ((16720, 16779), 'logging.basicConfig', 'logging.basicConfig', ([], {'stream': 'sys.stdout', 'level': 'logging.DEBUG'}), '(stream=sys.stdout, level=logging.DEBUG)\n', (16739, 16779), False, 'import logging\n'), ((1625, 1640), 'torch.no_grad', 'torch.no_grad', ([], {}), '()\n', (1638, 1640), False, 'import torch\n'), ((14622, 14671), 'os.path.join', 'os.path.join', (['args.lora_path', '"""pytorch_model.bin"""'], {}), "(args.lora_path, 'pytorch_model.bin')\n", (14634, 14671), False, 'import os\n'), ((14707, 14739), 'os.path.exists', 'os.path.exists', (['pytorch_bin_path'], {}), '(pytorch_bin_path)\n', (14721, 14739), False, 'import os\n'), ((15075, 15108), 'torch.backends.mps.is_available', 'torch.backends.mps.is_available', ([], {}), '()\n', (15106, 15108), False, 'import torch\n'), ((15184, 15303), 'transformers.LlamaForCausalLM.from_pretrained', 'LlamaForCausalLM.from_pretrained', (['BASE_MODEL'], {'load_in_8bit': 'LOAD_8BIT', 'torch_dtype': 'torch.float16', 'device_map': "{'': 0}"}), "(BASE_MODEL, load_in_8bit=LOAD_8BIT,\n    torch_dtype=torch.float16, device_map={'': 0})\n", (15216, 15303), False, 'from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n'), ((16696, 16716), 'torch.compile', 'torch.compile', (['model'], {}), '(model)\n', (16709, 16716), False, 'import torch\n'), ((16811, 16851), 'logging.StreamHandler', 'logging.StreamHandler', ([], {'stream': 'sys.stdout'}), '(stream=sys.stdout)\n', (16832, 16851), False, 'import logging\n'), ((17490, 17739), 'transformers.GenerationConfig', 'GenerationConfig', ([], {'temperature': 'temperature', 'top_p': 'top_p', 'top_k': 'top_k', 'num_beams': 'num_beams', 'bos_token_id': '(1)', 'eos_token_id': '(2)', 'pad_token_id': '(0)', 'max_new_tokens': 'max_new_tokens', 'min_new_tokens': 'min_new_tokens', 'repetition_penalty': 'repetition_penalty'}), '(temperature=temperature, top_p=top_p, top_k=top_k,\n    num_beams=num_beams, bos_token_id=1, eos_token_id=2, pad_token_id=0,\n    max_new_tokens=max_new_tokens, min_new_tokens=min_new_tokens,\n    repetition_penalty=repetition_penalty)\n', (17506, 17739), False, 'from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n'), ((19097, 19120), 'faiss.IndexFlatL2', 'faiss.IndexFlatL2', (['(1536)'], {}), '(1536)\n', (19114, 19120), False, 'import faiss\n'), ((19126, 19179), 'faiss.write_index', 'faiss.write_index', (['faiss_index', 'faiss_index_save_path'], {}), '(faiss_index, faiss_index_save_path)\n', (19143, 19179), False, 'import faiss\n'), ((19193, 19324), 'llama_index.GPTFaissIndex.load_from_disk', 'GPTFaissIndex.load_from_disk', ([], {'save_path': '"""clash-index.json"""', 'faiss_index': 'faiss_index_save_path', 'service_context': 'service_context'}), "(save_path='clash-index.json', faiss_index=\n    faiss_index_save_path, service_context=service_context)\n", (19221, 19324), False, 'from llama_index import GPTKeywordTableIndex, GPTSimpleVectorIndex, GPTListIndex, GPTTreeIndex, GPTFaissIndex\n'), ((3383, 3415), 'copy.deepcopy', 'copy.deepcopy', (['generation_config'], {}), '(generation_config)\n', (3396, 3415), False, 'import copy\n'), ((6104, 6429), 'transformers.generation.beam_search.BeamSearchScorer', 'BeamSearchScorer', ([], {'batch_size': 'batch_size', 'num_beams': 'generation_config.num_beams', 'device': 'input_ids.device', 'length_penalty': 'generation_config.length_penalty', 'do_early_stopping': 'generation_config.early_stopping', 'num_beam_hyps_to_keep': 'generation_config.num_return_sequences', 'max_length': 'generation_config.max_length'}), '(batch_size=batch_size, num_beams=generation_config.\n    num_beams, device=input_ids.device, length_penalty=generation_config.\n    length_penalty, do_early_stopping=generation_config.early_stopping,\n    num_beam_hyps_to_keep=generation_config.num_return_sequences,\n    max_length=generation_config.max_length)\n', (6120, 6429), False, 'from transformers.generation.beam_search import BeamSearchScorer\n'), ((7160, 7245), 'torch.zeros', 'torch.zeros', (['(batch_size, num_beams)'], {'dtype': 'torch.float', 'device': 'input_ids.device'}), '((batch_size, num_beams), dtype=torch.float, device=input_ids.device\n    )\n', (7171, 7245), False, 'import torch\n'), ((11738, 11774), 'peft.LoraConfig.from_pretrained', 'LoraConfig.from_pretrained', (['model_id'], {}), '(model_id)\n', (11764, 11774), False, 'from peft import PeftModelForCausalLM, LoraConfig\n'), ((12718, 12768), 'peft.utils.set_peft_model_state_dict', 'set_peft_model_state_dict', (['model', 'adapters_weights'], {}), '(model, adapters_weights)\n', (12743, 12768), False, 'from peft.utils import PeftType, set_peft_model_state_dict\n'), ((14549, 14578), 'os.path.exists', 'os.path.exists', (['lora_bin_path'], {}), '(lora_bin_path)\n', (14563, 14578), False, 'import os\n'), ((14749, 14791), 'os.rename', 'os.rename', (['pytorch_bin_path', 'lora_bin_path'], {}), '(pytorch_bin_path, lora_bin_path)\n', (14758, 14791), False, 'import os\n'), ((14800, 14919), 'warnings.warn', 'warnings.warn', (['"""The file name of the lora checkpoint\'pytorch_model.bin\' is replaced with \'adapter_model.bin\'"""'], {}), '(\n    "The file name of the lora checkpoint\'pytorch_model.bin\' is replaced with \'adapter_model.bin\'"\n    )\n', (14813, 14919), False, 'import warnings\n'), ((15504, 15604), 'transformers.LlamaForCausalLM.from_pretrained', 'LlamaForCausalLM.from_pretrained', (['BASE_MODEL'], {'device_map': "{'': device}", 'torch_dtype': 'torch.float16'}), "(BASE_MODEL, device_map={'': device},\n    torch_dtype=torch.float16)\n", (15536, 15604), False, 'from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n'), ((15811, 15908), 'transformers.LlamaForCausalLM.from_pretrained', 'LlamaForCausalLM.from_pretrained', (['BASE_MODEL'], {'device_map': "{'': device}", 'low_cpu_mem_usage': '(True)'}), "(BASE_MODEL, device_map={'': device},\n    low_cpu_mem_usage=True)\n", (15843, 15908), False, 'from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig\n'), ((16780, 16799), 'logging.getLogger', 'logging.getLogger', ([], {}), '()\n', (16797, 16799), False, 'import logging\n'), ((2134, 2162), 'transformers.deepspeed.is_deepspeed_zero3_enabled', 'is_deepspeed_zero3_enabled', ([], {}), '()\n', (2160, 2162), False, 'from transformers.deepspeed import is_deepspeed_zero3_enabled\n'), ((2584, 2651), 'torch.cat', 'torch.cat', (["(prefix_attention_mask, kwargs['attention_mask'])"], {'dim': '(1)'}), "((prefix_attention_mask, kwargs['attention_mask']), dim=1)\n", (2593, 2651), False, 'import torch\n'), ((2751, 2863), 'warnings.warn', 'warnings.warn', (['"""Position ids are not supported for parameter efficient tuning. Ignoring position ids."""'], {}), "(\n    'Position ids are not supported for parameter efficient tuning. Ignoring position ids.'\n    )\n", (2764, 2863), False, 'import warnings\n'), ((2997, 3112), 'warnings.warn', 'warnings.warn', (['"""Token type ids are not supported for parameter efficient tuning. Ignoring token type ids"""'], {}), "(\n    'Token type ids are not supported for parameter efficient tuning. Ignoring token type ids'\n    )\n", (3010, 3112), False, 'import warnings\n'), ((3995, 4306), 'warnings.warn', 'warnings.warn', (['f"""Using `max_length`\'s default ({generation_config.max_length}) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation."""', 'UserWarning'], {}), '(\n    f"Using `max_length`\'s default ({generation_config.max_length}) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation."\n    , UserWarning)\n', (4008, 4306), False, 'import warnings\n'), ((5118, 5139), 'transformers.generation.utils.LogitsProcessorList', 'LogitsProcessorList', ([], {}), '()\n', (5137, 5139), False, 'from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList, GenerationMixin\n'), ((5272, 5294), 'transformers.generation.utils.StoppingCriteriaList', 'StoppingCriteriaList', ([], {}), '()\n', (5292, 5294), False, 'from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList, GenerationMixin\n'), ((8817, 8869), 'torch.nn.functional.log_softmax', 'nn.functional.log_softmax', (['next_token_logits'], {'dim': '(-1)'}), '(next_token_logits, dim=-1)\n', (8842, 8869), False, 'from torch import nn\n'), ((9542, 9620), 'torch.topk', 'torch.topk', (['next_token_scores', '(2 * num_beams)'], {'dim': '(1)', 'largest': '(True)', 'sorted': '(True)'}), '(next_token_scores, 2 * num_beams, dim=1, largest=True, sorted=True)\n', (9552, 9620), False, 'import torch\n'), ((9678, 9735), 'torch.div', 'torch.div', (['next_tokens', 'vocab_size'], {'rounding_mode': '"""floor"""'}), "(next_tokens, vocab_size, rounding_mode='floor')\n", (9687, 9735), False, 'import torch\n'), ((11850, 11884), 'accelerate.hooks.remove_hook_from_submodules', 'remove_hook_from_submodules', (['model'], {}), '(model)\n', (11877, 11884), False, 'from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules\n'), ((12005, 12048), 'os.path.join', 'os.path.join', (['model_id', '"""adapter_model.bin"""'], {}), "(model_id, 'adapter_model.bin')\n", (12017, 12048), False, 'import os\n'), ((12074, 12117), 'os.path.join', 'os.path.join', (['model_id', '"""adapter_model.bin"""'], {}), "(model_id, 'adapter_model.bin')\n", (12086, 12117), False, 'import os\n'), ((13593, 13637), 'accelerate.dispatch_model', 'dispatch_model', (['model'], {'device_map': 'device_map'}), '(model, device_map=device_map)\n', (13607, 13637), False, 'from accelerate import dispatch_model, infer_auto_device_map\n'), ((13657, 13694), 'accelerate.hooks.AlignDevicesHook', 'AlignDevicesHook', ([], {'io_same_device': '(True)'}), '(io_same_device=True)\n', (13673, 13694), False, 'from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules\n'), ((18251, 18274), 'llama_index.LLMPredictor', 'LLMPredictor', ([], {'llm': 'model'}), '(llm=model)\n', (18263, 18274), False, 'from llama_index import LLMPredictor\n'), ((18298, 18357), 'llama_index.PromptHelper', 'PromptHelper', (['max_input_size', 'num_output', 'max_chunk_overlap'], {}), '(max_input_size, num_output, max_chunk_overlap)\n', (18310, 18357), False, 'from llama_index import PromptHelper, SimpleDirectoryReader\n'), ((18376, 18426), 'llama_index.SimpleDirectoryReader', 'SimpleDirectoryReader', (['"""Chinese-Vicuna/index-docs"""'], {}), "('Chinese-Vicuna/index-docs')\n", (18397, 18426), False, 'from llama_index import PromptHelper, SimpleDirectoryReader\n'), ((2167, 2184), 'torch.distributed.world_size', 'dist.world_size', ([], {}), '()\n', (2182, 2184), True, 'import torch.distributed as dist\n'), ((7914, 7976), 'torch.distributed.all_reduce', 'dist.all_reduce', (['this_peer_finished_flag'], {'op': 'dist.ReduceOp.SUM'}), '(this_peer_finished_flag, op=dist.ReduceOp.SUM)\n', (7929, 7976), True, 'import torch.distributed as dist\n'), ((12176, 12222), 'huggingface_hub.hf_hub_download', 'hf_hub_download', (['model_id', '"""adapter_model.bin"""'], {}), "(model_id, 'adapter_model.bin')\n", (12191, 12222), False, 'from huggingface_hub import hf_hub_download\n'), ((13079, 13223), 'accelerate.utils.get_balanced_memory', 'get_balanced_memory', (['model'], {'max_memory': 'max_memory', 'no_split_module_classes': 'no_split_module_classes', 'low_zero': "(device_map == 'balanced_low_0')"}), "(model, max_memory=max_memory, no_split_module_classes=\n    no_split_module_classes, low_zero=device_map == 'balanced_low_0')\n", (13098, 13223), False, 'from accelerate.utils import get_balanced_memory\n'), ((13393, 13498), 'accelerate.infer_auto_device_map', 'infer_auto_device_map', (['model'], {'max_memory': 'max_memory', 'no_split_module_classes': 'no_split_module_classes'}), '(model, max_memory=max_memory, no_split_module_classes\n    =no_split_module_classes)\n', (13414, 13498), False, 'from accelerate import dispatch_model, infer_auto_device_map\n'), ((13772, 13820), 'accelerate.hooks.add_hook_to_module', 'add_hook_to_module', (['model.base_model.model', 'hook'], {}), '(model.base_model.model, hook)\n', (13790, 13820), False, 'from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules\n'), ((13855, 13904), 'accelerate.hooks.remove_hook_from_submodules', 'remove_hook_from_submodules', (['model.prompt_encoder'], {}), '(model.prompt_encoder)\n', (13882, 13904), False, 'from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules\n'), ((13921, 13963), 'accelerate.hooks.add_hook_to_module', 'add_hook_to_module', (['model.base_model', 'hook'], {}), '(model.base_model, hook)\n', (13939, 13963), False, 'from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules\n'), ((2406, 2483), 'torch.ones', 'torch.ones', (["kwargs['input_ids'].shape[0]", 'self.peft_config.num_virtual_tokens'], {}), "(kwargs['input_ids'].shape[0], self.peft_config.num_virtual_tokens)\n", (2416, 2483), False, 'import torch\n'), ((7733, 7781), 'torch.tensor', 'torch.tensor', (['(0.0 if this_peer_finished else 1.0)'], {}), '(0.0 if this_peer_finished else 1.0)\n', (7745, 7781), False, 'import torch\n'), ((12611, 12636), 'torch.cuda.is_available', 'torch.cuda.is_available', ([], {}), '()\n', (12634, 12636), False, 'import torch\n'), ((20831, 20919), 'gradio.components.Textbox', 'gr.components.Textbox', ([], {'lines': '(2)', 'label': '"""Input"""', 'placeholder': '"""Tell me about alpacas."""'}), "(lines=2, label='Input', placeholder=\n    'Tell me about alpacas.')\n", (20852, 20919), True, 'import gradio as gr\n'), ((20946, 21020), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(0)', 'maximum': '(1)', 'value': '(0.1)', 'label': '"""Temperature"""'}), "(minimum=0, maximum=1, value=0.1, label='Temperature')\n", (20966, 21020), True, 'import gradio as gr\n'), ((21030, 21099), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(0)', 'maximum': '(1)', 'value': '(0.75)', 'label': '"""Top p"""'}), "(minimum=0, maximum=1, value=0.75, label='Top p')\n", (21050, 21099), True, 'import gradio as gr\n'), ((21109, 21186), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(0)', 'maximum': '(100)', 'step': '(1)', 'value': '(40)', 'label': '"""Top k"""'}), "(minimum=0, maximum=100, step=1, value=40, label='Top k')\n", (21129, 21186), True, 'import gradio as gr\n'), ((21196, 21283), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(1)', 'maximum': '(10)', 'step': '(1)', 'value': '(4)', 'label': '"""Beams Number"""'}), "(minimum=1, maximum=10, step=1, value=4, label=\n    'Beams Number')\n", (21216, 21283), True, 'import gradio as gr\n'), ((21288, 21381), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(1)', 'maximum': '(2000)', 'step': '(1)', 'value': '(256)', 'label': '"""Max New Tokens"""'}), "(minimum=1, maximum=2000, step=1, value=256, label=\n    'Max New Tokens')\n", (21308, 21381), True, 'import gradio as gr\n'), ((21408, 21498), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(1)', 'maximum': '(100)', 'step': '(1)', 'value': '(1)', 'label': '"""Min New Tokens"""'}), "(minimum=1, maximum=100, step=1, value=1, label=\n    'Min New Tokens')\n", (21428, 21498), True, 'import gradio as gr\n'), ((21525, 21626), 'gradio.components.Slider', 'gr.components.Slider', ([], {'minimum': '(0.1)', 'maximum': '(10.0)', 'step': '(0.1)', 'value': '(1.0)', 'label': '"""Repetition Penalty"""'}), "(minimum=0.1, maximum=10.0, step=0.1, value=1.0, label=\n    'Repetition Penalty')\n", (21545, 21626), True, 'import gradio as gr\n'), ((21674, 21717), 'gradio.inputs.Textbox', 'gr.inputs.Textbox', ([], {'lines': '(15)', 'label': '"""Output"""'}), "(lines=15, label='Output')\n", (21691, 21717), True, 'import gradio as gr\n')]