


'''simple docstring'''

import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


a_  :				Union[str, Any]						= ["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]





class      snake_case    (    lowercase			):





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase=None			,  UpperCamelCase=1     ):







   """simple docstring"""
   lowerCamelCase_      =  tokenizer
   lowerCamelCase_      =  dataset
   lowerCamelCase_      =  len(UpperCamelCase     ) if n_tasks is None else n_tasks
   lowerCamelCase_      =  n_copies







  def __iter__(    self     ):







   """simple docstring"""
   lowerCamelCase_      =  []
   for task in range(self.n_tasks     ):
    # without strip, the model generate commented codes ...
    prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip()     )
   lowerCamelCase_      =  self.tokenizer(UpperCamelCase			,  padding=UpperCamelCase			,  return_tensors="pt"     )
   for task in range(self.n_tasks     ):
    for _ in range(self.n_copies     ):
     yield {
         "ids": outputs.input_ids[task],
         "task_id": task,
         "input_len": outputs.attention_mask[task].sum(),
     }





class      snake_case    (    lowercase			):





  """simple docstring"""




  def __init__(    self			,  UpperCamelCase			,  UpperCamelCase			,  UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  start_length
   lowerCamelCase_      =  eof_strings
   lowerCamelCase_      =  tokenizer







  def __call__(    self			,  UpperCamelCase			,  UpperCamelCase			,  **UpperCamelCase     ):







   """simple docstring"""
   lowerCamelCase_      =  self.tokenizer.batch_decode(input_ids[:, self.start_length :]     )
   lowerCamelCase_      =  []
   for decoded_generation in decoded_generations:
    done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings     )     )
   return all(UpperCamelCase     )





def __snake_case					(      UpperCAmelCase_						:						Optional[Any]		):
 lowerCamelCase_      =  re.split("(%s)" % "|".join(UpperCAmelCase_		)					,				UpperCAmelCase_		)
 # last string should be ""
 return "".join(string_list[:-2]		)





def __snake_case					(      UpperCAmelCase_						:						Dict					,				UpperCAmelCase_						:						int					,				UpperCAmelCase_						:						Any					,				UpperCAmelCase_						:						Dict					,				UpperCAmelCase_						:						int					,				UpperCAmelCase_						:						str=20					,				**UpperCAmelCase_						:						Optional[Any]		):
 lowerCamelCase_      =  defaultdict(UpperCAmelCase_		)  # dict of list of generated tokens
 for step, batch in tqdm(enumerate(UpperCAmelCase_		)		):
  with torch.no_grad():
   lowerCamelCase_      =  batch["ids"].shape[-1]
   lowerCamelCase_      =  accelerator.unwrap_model(UpperCAmelCase_		).generate(
       input_ids=batch["ids"][:, : batch["input_len"]]					,				num_return_sequences=UpperCAmelCase_					,				**UpperCAmelCase_		)
   # each task is generated batch_size times
   lowerCamelCase_      =  batch["task_id"].repeat(UpperCAmelCase_		)
   lowerCamelCase_      =  accelerator.pad_across_processes(
       UpperCAmelCase_					,				dim=1					,				pad_index=tokenizer.pad_token_id		)

   lowerCamelCase_ ,lowerCamelCase_      =  accelerator.gather((generated_tokens, generated_tasks)		)
   lowerCamelCase_      =  generated_tokens.cpu().numpy()
   lowerCamelCase_      =  generated_tasks.cpu().numpy()

   for task, generated_tokens in zip(UpperCAmelCase_					,				UpperCAmelCase_		):
    gen_token_dict[task].append(UpperCAmelCase_		)

 lowerCamelCase_      =  [[] for _ in range(UpperCAmelCase_		)]
 for task, generated_tokens in gen_token_dict.items():
  for s in generated_tokens:
   lowerCamelCase_      =  tokenizer.decode(UpperCAmelCase_					,				skip_special_tokens=UpperCAmelCase_					,				clean_up_tokenization_spaces=UpperCAmelCase_		)
   code_gens[task].append(remove_last_block(UpperCAmelCase_		)		)
 return code_gens





def __snake_case					(      ):
 # Setup configuration
 lowerCamelCase_      =  HfArgumentParser(UpperCAmelCase_		)
 lowerCamelCase_      =  parser.parse_args()

 transformers.logging.set_verbosity_error()
 # enables code execution in code_eval metric
 lowerCamelCase_      =  args.HF_ALLOW_CODE_EVAL
 # make sure tokenizer plays nice with multiprocessing
 lowerCamelCase_      =  "false"

 if args.num_workers is None:
  lowerCamelCase_      =  multiprocessing.cpu_count()

 # Use dataset load to feed to accelerate
 lowerCamelCase_      =  Accelerator()
 set_seed(args.seed					,				device_specific=UpperCAmelCase_		)

 # Load model and tokenizer
 lowerCamelCase_      =  AutoTokenizer.from_pretrained(args.model_ckpt		)
 lowerCamelCase_      =  tokenizer.eos_token
 lowerCamelCase_      =  AutoModelForCausalLM.from_pretrained(args.model_ckpt		)

 # Generation settings
 lowerCamelCase_      =  {
     "do_sample": args.do_sample,
     "temperature": args.temperature,
     "max_new_tokens": args.max_new_tokens,
     "top_p": args.top_p,
     "top_k": args.top_k,
     "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0					,				UpperCAmelCase_					,				UpperCAmelCase_		)]		),
 }

 # Load evaluation dataset and metric
 lowerCamelCase_      =  load_dataset("openai_humaneval"		)
 lowerCamelCase_      =  load_metric("code_eval"		)

 lowerCamelCase_      =  args.num_tasks if args.num_tasks is not None else len(human_eval["test"]		)
 lowerCamelCase_      =  args.n_samples // args.batch_size

 lowerCamelCase_      =  TokenizedDataset(UpperCAmelCase_					,				human_eval["test"]					,				n_copies=UpperCAmelCase_					,				n_tasks=UpperCAmelCase_		)
 # do not confuse args.batch_size, which is actually the num_return_sequences
 lowerCamelCase_      =  DataLoader(UpperCAmelCase_					,				batch_size=1		)

 # Run a quick test to see if code evaluation is enabled
 try:
  lowerCamelCase_      =  code_eval_metric.compute(references=[""]					,				predictions=[[""]]		)
 except ValueError as exception:
  print(
      "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
      " flag to enable code evaluation."		)
  raise exception

 lowerCamelCase_ ,lowerCamelCase_      =  accelerator.prepare(UpperCAmelCase_					,				UpperCAmelCase_		)

 lowerCamelCase_      =  complete_code(
     UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_					,				n_tasks=UpperCAmelCase_					,				batch_size=args.batch_size					,				**UpperCAmelCase_					,				)

 if accelerator.is_main_process:
  lowerCamelCase_      =  []

  for task in tqdm(range(UpperCAmelCase_		)		):
   lowerCamelCase_      =  human_eval["test"][task]["test"]
   lowerCamelCase_      =  F'''check({human_eval["test"][task]["entry_point"]})'''
   references.append("\n" + test_func + "\n" + entry_point		)

  # Evaluate completions with "code_eval" metric
  lowerCamelCase_ ,lowerCamelCase_      =  code_eval_metric.compute(
      references=UpperCAmelCase_					,				predictions=UpperCAmelCase_					,				num_workers=args.num_workers		)
  print(F'''Results: {pass_at_k}'''		)

  # Save results to json file
  with open(args.output_file					,				"w"		) as fp:
   json.dump(UpperCAmelCase_					,				UpperCAmelCase_		)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
     main()







'''simple docstring'''
# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCamelCase__    					=      {
    '''configuration_vivit''': ['''VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''VivitConfig'''],
}
try:
  if not is_vision_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__    					=      ['''VivitImageProcessor''']


try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCamelCase__    					=      [
      '''VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''VivitModel''',
      '''VivitPreTrainedModel''',
      '''VivitForVideoClassification''',
  ]


if TYPE_CHECKING:
  from .configuration_vivit import VIVIT_PRETRAINED_CONFIG_ARCHIVE_MAP, VivitConfig

  try:
    if not is_vision_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .image_processing_vivit import VivitImageProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_vivit import (
        VIVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
        VivitForVideoClassification,
        VivitModel,
        VivitPreTrainedModel,
    )


else:
  import sys

  UpperCamelCase__    					=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
