






import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


A__	:			List[str]			   = ['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				def __init__(     self						:    Optional[int],  lowerCamelCase						:    List[Any],  lowerCamelCase						:    Dict,  lowerCamelCase						:    str=None,  lowerCamelCase						:    Optional[int]=1  ):

								'''simple docstring'''
								lowercase__    =			tokenizer
								lowercase__    =			dataset
								lowercase__    =			len(lowerCamelCase  ) if n_tasks is None else n_tasks
								lowercase__    =			n_copies




				def __iter__(     self						:    Union[str, Any]  ):

								'''simple docstring'''
								lowercase__    =			[]
								for task in range(self.n_tasks  ):
												# without strip, the model generate commented codes ...
												prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()  )
								lowercase__    =			self.tokenizer(lowerCamelCase,  padding=lowerCamelCase,  return_tensors='''pt'''  )
								for task in range(self.n_tasks  ):
												for _ in range(self.n_copies  ):
																yield {
																    "ids": outputs.input_ids[task],
																    "task_id": task,
																    "input_len": outputs.attention_mask[task].sum(),
																}



class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				def __init__(     self						:    List[Any],  lowerCamelCase						:    str,  lowerCamelCase						:    Any,  lowerCamelCase						:    Union[str, Any]  ):

								'''simple docstring'''
								lowercase__    =			start_length
								lowercase__    =			eof_strings
								lowercase__    =			tokenizer




				def __call__(     self						:    Union[str, Any],  lowerCamelCase						:    str,  lowerCamelCase						:    Tuple,  **lowerCamelCase						:    Tuple  ):

								'''simple docstring'''
								lowercase__    =			self.tokenizer.batch_decode(input_ids[:, self.start_length :]  )
								lowercase__    =			[]
								for decoded_generation in decoded_generations:
												done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings  )  )
								return all(lowerCamelCase  )






def   a			(						lowerCamelCase_				):






				'''simple docstring'''
				lowercase__    =			re.split('''(%s)''' % '''|'''.join(__lowerCAmelCase				)		,   __lowerCAmelCase				)
				# last string should be ""
				return "".join(string_list[:-2]				)






def   a			(						lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_=20		,   **lowerCamelCase_				):






				'''simple docstring'''
				lowercase__    =			defaultdict(__lowerCAmelCase				)  # dict of list of generated tokens
				for step, batch in tqdm(enumerate(__lowerCAmelCase				)				):
								with torch.no_grad():
												lowercase__    =			batch["ids"].shape[-1]
												lowercase__    =			accelerator.unwrap_model(__lowerCAmelCase				).generate(
												    input_ids=batch['''ids'''][:, : batch['''input_len''']]		,   num_return_sequences=__lowerCAmelCase		,   **__lowerCAmelCase				)
												# each task is generated batch_size times
												lowercase__    =			batch["task_id"].repeat(__lowerCAmelCase				)
												lowercase__    =			accelerator.pad_across_processes(
												    __lowerCAmelCase		,   dim=1		,   pad_index=tokenizer.pad_token_id				)

												lowercase__    =			accelerator.gather((generated_tokens, generated_tasks)				)
												lowercase__    =			generated_tokens.cpu().numpy()
												lowercase__    =			generated_tasks.cpu().numpy()

												for task, generated_tokens in zip(__lowerCAmelCase		,   __lowerCAmelCase				):
																gen_token_dict[task].append(__lowerCAmelCase				)

				lowercase__    =			[[] for _ in range(__lowerCAmelCase				)]
				for task, generated_tokens in gen_token_dict.items():
								for s in generated_tokens:
												lowercase__    =			tokenizer.decode(__lowerCAmelCase		,   skip_special_tokens=__lowerCAmelCase		,   clean_up_tokenization_spaces=__lowerCAmelCase				)
												code_gens[task].append(remove_last_block(__lowerCAmelCase				)				)
				return code_gens






def   a			(						):






				'''simple docstring'''
				# Setup configuration
				lowercase__    =			HfArgumentParser(__lowerCAmelCase				)
				lowercase__    =			parser.parse_args()

				transformers.logging.set_verbosity_error()
				# enables code execution in code_eval metric
				lowercase__    =			args.HF_ALLOW_CODE_EVAL
				# make sure tokenizer plays nice with multiprocessing
				lowercase__    =			"false"

				if args.num_workers is None:
								lowercase__    =			multiprocessing.cpu_count()

				# Use dataset load to feed to accelerate
				lowercase__    =			Accelerator()
				set_seed(args.seed		,   device_specific=__lowerCAmelCase				)

				# Load model and tokenizer
				lowercase__    =			AutoTokenizer.from_pretrained(args.model_ckpt				)
				lowercase__    =			tokenizer.eos_token
				lowercase__    =			AutoModelForCausalLM.from_pretrained(args.model_ckpt				)

				# Generation settings
				lowercase__    =			{
				    "do_sample": args.do_sample,
				    "temperature": args.temperature,
				    "max_new_tokens": args.max_new_tokens,
				    "top_p": args.top_p,
				    "top_k": args.top_k,
				    "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0		,   __lowerCAmelCase		,   __lowerCAmelCase				)]				),
				}

				# Load evaluation dataset and metric
				lowercase__    =			load_dataset('''openai_humaneval'''				)
				lowercase__    =			load_metric('''code_eval'''				)

				lowercase__    =			args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']				)
				lowercase__    =			args.n_samples // args.batch_size

				lowercase__    =			TokenizedDataset(__lowerCAmelCase		,   human_eval['''test''']		,   n_copies=__lowerCAmelCase		,   n_tasks=__lowerCAmelCase				)
				# do not confuse args.batch_size, which is actually the num_return_sequences
				lowercase__    =			DataLoader(__lowerCAmelCase		,   batch_size=1				)

				# Run a quick test to see if code evaluation is enabled
				try:
								lowercase__    =			code_eval_metric.compute(references=['''''']		,   predictions=[['''''']]				)
				except ValueError as exception:
								print(
								    '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`'''
								    ''' flag to enable code evaluation.'''				)
								raise exception

				lowercase__    =			accelerator.prepare(__lowerCAmelCase		,   __lowerCAmelCase				)

				lowercase__    =			complete_code(
				    __lowerCAmelCase		,   __lowerCAmelCase		,   __lowerCAmelCase		,   __lowerCAmelCase		,   n_tasks=__lowerCAmelCase		,   batch_size=args.batch_size		,   **__lowerCAmelCase		,   )

				if accelerator.is_main_process:
								lowercase__    =			[]

								for task in tqdm(range(__lowerCAmelCase				)				):
												lowercase__    =			human_eval["test"][task]["test"]
												lowercase__    =			F"""check({human_eval['test'][task]['entry_point']})"""
												references.append('''\n''' + test_func + '''\n''' + entry_point				)

								# Evaluate completions with "code_eval" metric
								lowercase__    =			code_eval_metric.compute(
								    references=__lowerCAmelCase		,   predictions=__lowerCAmelCase		,   num_workers=args.num_workers				)
								print(F"""Results: {pass_at_k}"""				)

								# Save results to json file
								with open(args.output_file		,   '''w'''				) as fp:
												json.dump(__lowerCAmelCase		,   __lowerCAmelCase				)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
							main()












import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

lowerCamelCase	:							Union[str, Any]				=logging.get_logger(__name__)


lowerCamelCase	:							str				={
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

lowerCamelCase	:							str				={
    '''vocab_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'''},
    '''merges_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'''},
    '''tokenizer_config_file''': {
        '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'''
    },
}

lowerCamelCase	:							str				={'''facebook/blenderbot-3B''': 128}





class 			__a  (    A__       ):
 _lowerCAmelCase       :       Union[str, Any]        = VOCAB_FILES_NAMES
 _lowerCAmelCase       :       Dict        = PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase       :       List[Any]        = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase       :       int        = ['''input_ids''', '''attention_mask''']
 _lowerCAmelCase       :       Tuple        = BlenderbotTokenizer







 def __init__(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   Optional[int]=None							,   SCREAMING_SNAKE_CASE					:   Dict=None							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]=None							,   SCREAMING_SNAKE_CASE					:   Tuple="replace"							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]="<s>"							,   SCREAMING_SNAKE_CASE					:   str="</s>"							,   SCREAMING_SNAKE_CASE					:   str="</s>"							,   SCREAMING_SNAKE_CASE					:   Tuple="<s>"							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]="<unk>"							,   SCREAMING_SNAKE_CASE					:   str="<pad>"							,   SCREAMING_SNAKE_CASE					:   Union[str, Any]="<mask>"							,   SCREAMING_SNAKE_CASE					:   Any=False							,   SCREAMING_SNAKE_CASE					:   List[Any]=True							,   **SCREAMING_SNAKE_CASE					:   int							,   ):


      '''simple docstring'''






      super().__init__(
          SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE							,   tokenizer_file=SCREAMING_SNAKE_CASE							,   errors=SCREAMING_SNAKE_CASE							,   bos_token=SCREAMING_SNAKE_CASE							,   eos_token=SCREAMING_SNAKE_CASE							,   sep_token=SCREAMING_SNAKE_CASE							,   cls_token=SCREAMING_SNAKE_CASE							,   unk_token=SCREAMING_SNAKE_CASE							,   pad_token=SCREAMING_SNAKE_CASE							,   mask_token=SCREAMING_SNAKE_CASE							,   add_prefix_space=SCREAMING_SNAKE_CASE							,   trim_offsets=SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE							,   )

      UpperCamelCase__			:   List[Any]       				=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
      if pre_tok_state.get("add_prefix_space"							,   SCREAMING_SNAKE_CASE						) != add_prefix_space:
           UpperCamelCase__			:   str       				=  getattr(SCREAMING_SNAKE_CASE							,   pre_tok_state.pop("type"						)						)
           UpperCamelCase__			:   Union[str, Any]       				=  add_prefix_space
           UpperCamelCase__			:   Union[str, Any]       				=  pre_tok_class(**SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   Tuple       				=  add_prefix_space

      UpperCamelCase__			:   Optional[int]       				=  "post_processor"
      UpperCamelCase__			:   Union[str, Any]       				=  getattr(self.backend_tokenizer							,   SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)
      if tokenizer_component_instance:
           UpperCamelCase__			:   Tuple       				=  json.loads(tokenizer_component_instance.__getstate__()						)

           # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
           if "sep" in state:
                UpperCamelCase__			:   Tuple       				=  tuple(state["sep"]						)
           if "cls" in state:
                UpperCamelCase__			:   Optional[int]       				=  tuple(state["cls"]						)

           UpperCamelCase__			:   List[Any]       				=  False

           if state.get("add_prefix_space"							,   SCREAMING_SNAKE_CASE						) != add_prefix_space:
                UpperCamelCase__			:   str       				=  add_prefix_space
                UpperCamelCase__			:   Optional[Any]       				=  True

           if state.get("trim_offsets"							,   SCREAMING_SNAKE_CASE						) != trim_offsets:
                UpperCamelCase__			:   Optional[Any]       				=  trim_offsets
                UpperCamelCase__			:   Tuple       				=  True

           if changes_to_apply:
                UpperCamelCase__			:   Optional[int]       				=  getattr(SCREAMING_SNAKE_CASE							,   state.pop("type"						)						)
                UpperCamelCase__			:   Dict       				=  component_class(**SCREAMING_SNAKE_CASE						)
                setattr(self.backend_tokenizer							,   SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						)







 @property
 # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot
 def 					__lowercase					(		self					:   Union[str, Any]						):


      '''simple docstring'''






      if self._mask_token is None:
           if self.verbose:
                logger.error("Using mask_token, but it is not set yet."						)
           return None
      return str(self._mask_token						)







 @mask_token.setter
 def 					__lowercase					(		self					:   Union[str, Any]							,   SCREAMING_SNAKE_CASE					:   Optional[int]						):


      '''simple docstring'''






      UpperCamelCase__			:   Dict       				=  AddedToken(SCREAMING_SNAKE_CASE							,   lstrip=SCREAMING_SNAKE_CASE							,   rstrip=SCREAMING_SNAKE_CASE						) if isinstance(SCREAMING_SNAKE_CASE							,   SCREAMING_SNAKE_CASE						) else value
      UpperCamelCase__			:   str       				=  value







 def 					__lowercase					(		self					:   Any							,   *SCREAMING_SNAKE_CASE					:   List[str]							,   **SCREAMING_SNAKE_CASE					:   Tuple						):


      '''simple docstring'''






      UpperCamelCase__			:   List[Any]       				=  kwargs.get("is_split_into_words"							,   SCREAMING_SNAKE_CASE						)
      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Dict							,   *SCREAMING_SNAKE_CASE					:   int							,   **SCREAMING_SNAKE_CASE					:   Dict						):


      '''simple docstring'''






      UpperCamelCase__			:   Dict       				=  kwargs.get("is_split_into_words"							,   SCREAMING_SNAKE_CASE						)

      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*SCREAMING_SNAKE_CASE							,   **SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Union[str, Any]							,   SCREAMING_SNAKE_CASE					:   str							,   SCREAMING_SNAKE_CASE					:   Optional[str] = None						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[Any]       				=  self._tokenizer.model.save(SCREAMING_SNAKE_CASE							,   name=SCREAMING_SNAKE_CASE						)
      return tuple(SCREAMING_SNAKE_CASE						)







 def 					__lowercase					(		self					:   Optional[int]							,   SCREAMING_SNAKE_CASE					:   List[int]							,   SCREAMING_SNAKE_CASE					:   Optional[List[int]] = None						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[Any]       				=  [self.sep_token_id]
      UpperCamelCase__			:   int       				=  [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep						) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]







 def 					__lowercase					(		self					:   List[Any]							,   SCREAMING_SNAKE_CASE					:   List[int]							,   SCREAMING_SNAKE_CASE					:   Optional[List[int]] = None						):


      '''simple docstring'''






      return token_ids_a + [self.eos_token_id]







 def 					__lowercase					(		self					:   Union[str, Any]							,   SCREAMING_SNAKE_CASE					:   "Conversation"						):


      '''simple docstring'''






      UpperCamelCase__			:   Optional[Any]       				=  []
      for is_user, text in conversation.iter_texts():
           if is_user:
                # We need to space prefix as it's being done within blenderbot
                inputs.append(" " + text						)
           else:
                # Generated responses should contain them already.
                inputs.append(SCREAMING_SNAKE_CASE						)

      UpperCamelCase__			:   List[str]       				=  "  ".join(SCREAMING_SNAKE_CASE						)
      UpperCamelCase__			:   Optional[Any]       				=  self.encode(SCREAMING_SNAKE_CASE						)
      if len(SCREAMING_SNAKE_CASE						) > self.model_max_length:
           UpperCamelCase__			:   Dict       				=  input_ids[-self.model_max_length :]
           logger.warning(F'Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'						)
      return input_ids