





import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


UpperCAmelCase_     							= ['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']






class     lowercase__       (	__lowerCamelCase				):

   '''simple docstring'''


   def __init__(    self,		__magic_name__,		__magic_name__,		__magic_name__=None,		__magic_name__=1							) ->      Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      tokenizer
       UpperCamelCase__							:				List[str]          =      dataset
       UpperCamelCase__							:				Any          =      len(__magic_name__							) if n_tasks is None else n_tasks
       UpperCamelCase__							:				Dict          =      n_copies




   def __iter__(    self							) ->      int:






       """simple docstring"""
       UpperCamelCase__							:				Dict          =      []
       for task in range(self.n_tasks							):
           # without strip, the model generate commented codes ...
           prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()							)
       UpperCamelCase__							:				Any          =      self.tokenizer(__magic_name__,		padding=__magic_name__,		return_tensors='''pt'''							)
       for task in range(self.n_tasks							):
           for _ in range(self.n_copies							):
               yield {
                   "ids": outputs.input_ids[task],
                   "task_id": task,
                   "input_len": outputs.attention_mask[task].sum(),
               }






class     lowercase__       (	__lowerCamelCase				):

   '''simple docstring'''


   def __init__(    self,		__magic_name__,		__magic_name__,		__magic_name__							) ->      Optional[int]:






       """simple docstring"""
       UpperCamelCase__							:				Optional[Any]          =      start_length
       UpperCamelCase__							:				Any          =      eof_strings
       UpperCamelCase__							:				Dict          =      tokenizer




   def __call__(    self,		__magic_name__,		__magic_name__,		**__magic_name__							) ->      Union[str, Any]:






       """simple docstring"""
       UpperCamelCase__							:				Tuple          =      self.tokenizer.batch_decode(input_ids[:, self.start_length :]							)
       UpperCamelCase__							:				Union[str, Any]          =      []
       for decoded_generation in decoded_generations:
           done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings							)							)
       return all(__magic_name__							)




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[Any]      )      ->	Optional[Any]:
    UpperCamelCase__							:				Optional[Any]          =      re.split('''(%s)''' % '''|'''.join(__UpperCAmelCase      )		,   __UpperCAmelCase      )
    # last string should be ""
    return "".join(string_list[:-2]      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Dict		,   __UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			Any		,   __UpperCAmelCase:			Optional[int]		,   __UpperCAmelCase:			Union[str, Any]		,   __UpperCAmelCase:			Tuple=20		,   **__UpperCAmelCase:			Tuple      )      ->	Dict:
    UpperCamelCase__							:				Tuple          =      defaultdict(__UpperCAmelCase      )  # dict of list of generated tokens
    for step, batch in tqdm(enumerate(__UpperCAmelCase      )      ):
        with torch.no_grad():
            UpperCamelCase__							:				List[Any]          =      batch['''ids'''].shape[-1]
            UpperCamelCase__							:				Any          =      accelerator.unwrap_model(__UpperCAmelCase      ).generate(
                input_ids=batch['''ids'''][:, : batch['''input_len''']]		,   num_return_sequences=__UpperCAmelCase		,   **__UpperCAmelCase      )
            # each task is generated batch_size times
            UpperCamelCase__							:				Union[str, Any]          =      batch['''task_id'''].repeat(__UpperCAmelCase      )
            UpperCamelCase__							:				Any          =      accelerator.pad_across_processes(
                __UpperCAmelCase		,   dim=1		,   pad_index=tokenizer.pad_token_id      )

            UpperCamelCase__   ,UpperCamelCase__							:				Dict          =      accelerator.gather((generated_tokens, generated_tasks)      )
            UpperCamelCase__							:				Optional[Any]          =      generated_tokens.cpu().numpy()
            UpperCamelCase__							:				Union[str, Any]          =      generated_tasks.cpu().numpy()

            for task, generated_tokens in zip(__UpperCAmelCase		,   __UpperCAmelCase      ):
                gen_token_dict[task].append(__UpperCAmelCase      )

    UpperCamelCase__							:				List[Any]          =      [[] for _ in range(__UpperCAmelCase      )]
    for task, generated_tokens in gen_token_dict.items():
        for s in generated_tokens:
            UpperCamelCase__							:				Tuple          =      tokenizer.decode(__UpperCAmelCase		,   skip_special_tokens=__UpperCAmelCase		,   clean_up_tokenization_spaces=__UpperCAmelCase      )
            code_gens[task].append(remove_last_block(__UpperCAmelCase      )      )
    return code_gens




def 		lowerCAmelCase_    (  )      ->	Tuple:
    # Setup configuration
    UpperCamelCase__							:				Union[str, Any]          =      HfArgumentParser(__UpperCAmelCase      )
    UpperCamelCase__							:				List[Any]          =      parser.parse_args()

    transformers.logging.set_verbosity_error()
    # enables code execution in code_eval metric
    UpperCamelCase__							:				int          =      args.HF_ALLOW_CODE_EVAL
    # make sure tokenizer plays nice with multiprocessing
    UpperCamelCase__							:				Any          =      '''false'''

    if args.num_workers is None:
        UpperCamelCase__							:				int          =      multiprocessing.cpu_count()

    # Use dataset load to feed to accelerate
    UpperCamelCase__							:				Union[str, Any]          =      Accelerator()
    set_seed(args.seed		,   device_specific=__UpperCAmelCase      )

    # Load model and tokenizer
    UpperCamelCase__							:				List[Any]          =      AutoTokenizer.from_pretrained(args.model_ckpt      )
    UpperCamelCase__							:				List[str]          =      tokenizer.eos_token
    UpperCamelCase__							:				List[str]          =      AutoModelForCausalLM.from_pretrained(args.model_ckpt      )

    # Generation settings
    UpperCamelCase__							:				Any          =      {
        '''do_sample''': args.do_sample,
        '''temperature''': args.temperature,
        '''max_new_tokens''': args.max_new_tokens,
        '''top_p''': args.top_p,
        '''top_k''': args.top_k,
        '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0		,   __UpperCAmelCase		,   __UpperCAmelCase      )]      ),
    }

    # Load evaluation dataset and metric
    UpperCamelCase__							:				int          =      load_dataset('''openai_humaneval'''      )
    UpperCamelCase__							:				Dict          =      load_metric('''code_eval'''      )

    UpperCamelCase__							:				Optional[int]          =      args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']      )
    UpperCamelCase__							:				Tuple          =      args.n_samples // args.batch_size

    UpperCamelCase__							:				str          =      TokenizedDataset(__UpperCAmelCase		,   human_eval['''test''']		,   n_copies=__UpperCAmelCase		,   n_tasks=__UpperCAmelCase      )
    # do not confuse args.batch_size, which is actually the num_return_sequences
    UpperCamelCase__							:				Optional[int]          =      DataLoader(__UpperCAmelCase		,   batch_size=1      )

    # Run a quick test to see if code evaluation is enabled
    try:
        UpperCamelCase__							:				Optional[Any]          =      code_eval_metric.compute(references=['''''']		,   predictions=[['''''']]      )
    except ValueError as exception:
        print(
            '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
            ''' flag to enable code evaluation.'''      )
        raise exception

    UpperCamelCase__   ,UpperCamelCase__							:				Dict          =      accelerator.prepare(__UpperCAmelCase		,   __UpperCAmelCase      )

    UpperCamelCase__							:				Tuple          =      complete_code(
        __UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase		,   n_tasks=__UpperCAmelCase		,   batch_size=args.batch_size		,   **__UpperCAmelCase		,   )

    if accelerator.is_main_process:
        UpperCamelCase__							:				List[str]          =      []

        for task in tqdm(range(__UpperCAmelCase      )      ):
            UpperCamelCase__							:				List[Any]          =      human_eval['''test'''][task]['''test''']
            UpperCamelCase__							:				Tuple          =      f"check({human_eval['test'][task]['entry_point']})"
            references.append('''\n''' + test_func + '''\n''' + entry_point      )

        # Evaluate completions with "code_eval" metric
        UpperCamelCase__   ,UpperCamelCase__							:				Tuple          =      code_eval_metric.compute(
            references=__UpperCAmelCase		,   predictions=__UpperCAmelCase		,   num_workers=args.num_workers      )
        print(f"Results: {pass_at_k}"      )

        # Save results to json file
        with open(args.output_file		,   '''w'''      ) as fp:
            json.dump(__UpperCAmelCase		,   __UpperCAmelCase      )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
  main()








def 		lowerCAmelCase_    (  __UpperCAmelCase:			int      )      ->	int:
    if not isinstance(__UpperCAmelCase		,   __UpperCAmelCase      ) or number < 0:
        raise ValueError('''Input must be a non-negative integer'''      )

    UpperCamelCase__							:				Optional[Any]          =      0
    while number:
        # This way we arrive at next set bit (next 1) instead of looping
        # through each bit and checking for 1s hence the
        # loop won't run 32 times it will only run the number of `1` times
        number &= number - 1
        count += 1
    return count


if __name__ == "__main__":
  import doctest

  doctest.testmod()

