



import logging
import os
import time
from argparse import Namespace
from pathlib import Path

import datasets
import torch
from accelerate import Accelerator, DistributedType
from accelerate.utils import ProjectConfiguration
from arguments import TrainingArguments
from datasets import load_dataset
from huggingface_hub import Repository
from torch.optim import AdamW
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from torch.utils.data.datapipes.iter.combinatorics import ShufflerIterDataPipe

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, get_scheduler, set_seed





class 		UpperCAmelCase    (    snake_case_					):





   def __init__(						self							,  _lowerCAmelCase							,  _lowerCAmelCase							,  _lowerCAmelCase=False							,  _lowerCAmelCase=1_024							,  _lowerCAmelCase=1_024							,  _lowerCAmelCase=3.6							,  _lowerCAmelCase=False							,  ):
         _lowerCAmelCase			      =	tokenizer
         _lowerCAmelCase			      =	tokenizer.bos_token_id
         _lowerCAmelCase			      =	dataset
         _lowerCAmelCase			      =	seq_length
         _lowerCAmelCase			      =	0
         _lowerCAmelCase			      =	infinite
         _lowerCAmelCase			      =	0
         _lowerCAmelCase			      =	tokenized

         if self.tokenized:
               _lowerCAmelCase			      =	seq_length * num_of_sequences
               _lowerCAmelCase			      =	'''input_ids'''
         else:
               _lowerCAmelCase			      =	seq_length * chars_per_token * num_of_sequences
               _lowerCAmelCase			      =	'''content'''





   def __iter__(						self      ):
         _lowerCAmelCase			      =	iter(self.dataset      )
         _lowerCAmelCase			      =	True
         while more_examples:
               _lowerCAmelCase      , _lowerCAmelCase			      =	[], 0
               while True:
                     if buffer_len >= self.max_buffer_size:
                           break
                     try:
                           buffer.append(next(_lowerCAmelCase      )[self.content_field]      )
                           buffer_len += len(buffer[-1]      )
                     except StopIteration:
                           if self.infinite:
                                 _lowerCAmelCase			      =	iter(self.dataset      )
                                 self.epoch += 1
                                 logger.info(F'''Dataset epoch: {self.epoch}'''      )
                           else:
                                 _lowerCAmelCase			      =	False
                                 break
               if self.tokenized:
                     _lowerCAmelCase			      =	buffer
               else:
                     _lowerCAmelCase			      =	self.tokenizer(_lowerCAmelCase							,  truncation=_lowerCAmelCase      )['''input_ids''']
               _lowerCAmelCase			      =	[]
               for tokenized_input in tokenized_inputs:
                     all_token_ids.extend(tokenized_input + [self.concat_token_id]      )
               for i in range(0							,  len(_lowerCAmelCase      )							,  self.seq_length      ):
                     _lowerCAmelCase			      =	all_token_ids[i : i + self.seq_length]
                     if len(_lowerCAmelCase      ) == self.seq_length:
                           self.current_size += 1
                           yield torch.tensor(_lowerCAmelCase      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase=1_000      ):
         return ShufflerIterDataPipe(self							,  buffer_size=_lowerCAmelCase      )






def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Union[str, Any]					)->str:
      _lowerCAmelCase			      =	args.model_ckpt.split('''/'''					)[-1]
      _lowerCAmelCase			      =	logging.getLogger(__name__					)
      _lowerCAmelCase			      =	Path(args.save_dir					) / '''log/'''
      log_dir.mkdir(exist_ok=_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	f'''debug_{accelerator.process_index}.log'''
      logging.basicConfig(
          format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''  ,					datefmt='''%m/%d/%Y %H:%M:%S'''  ,					level=logging.INFO  ,					handlers=[logging.FileHandler(log_dir / filename					), logging.StreamHandler()]  ,					)
      if accelerator.is_main_process:  # we only want to setup logging once
            accelerator.init_trackers(_SCREAMING_SNAKE_CASE  ,					vars(_SCREAMING_SNAKE_CASE					)					)
            _lowerCAmelCase			      =	accelerator.trackers[0].run.name
            logger.setLevel(logging.INFO					)
            datasets.utils.logging.set_verbosity_info()
            transformers.utils.logging.set_verbosity_info()
      else:
            _lowerCAmelCase			      =	''''''
            logger.setLevel(logging.ERROR					)
            datasets.utils.logging.set_verbosity_error()
            transformers.utils.logging.set_verbosity_error()
      return logger, run_name
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Any					)->int:
      _lowerCAmelCase			      =	{'''streaming''': True}
      _lowerCAmelCase			      =	load_dataset(args.dataset_name_train  ,					split='''train'''  ,					**_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	train_data.shuffle(buffer_size=args.shuffle_buffer  ,					seed=args.seed					)
      _lowerCAmelCase			      =	load_dataset(args.dataset_name_valid  ,					split='''train'''  ,					**_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	ConstantLengthDataset(
          _SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE  ,					infinite=_SCREAMING_SNAKE_CASE  ,					seq_length=args.seq_length  ,					tokenized=args.tokenized					)
      _lowerCAmelCase			      =	ConstantLengthDataset(
          _SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE  ,					infinite=_SCREAMING_SNAKE_CASE  ,					seq_length=args.seq_length  ,					tokenized=args.tokenized					)
      _lowerCAmelCase			      =	train_dataset.shuffle(buffer_size=args.shuffle_buffer					)
      _lowerCAmelCase			      =	DataLoader(_SCREAMING_SNAKE_CASE  ,					batch_size=args.train_batch_size  ,					shuffle=_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	DataLoader(_SCREAMING_SNAKE_CASE  ,					batch_size=args.valid_batch_size					)
      return train_dataloader, eval_dataloader
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Any  ,					_SCREAMING_SNAKE_CASE :						int  ,					_SCREAMING_SNAKE_CASE :						Any=["bias", "ln_1.weight", "ln_2.weight", "ln_f.weight"]					)->Any:
      _lowerCAmelCase      , _lowerCAmelCase			      =	[], []
      for n, p in model.named_parameters():
            if any(nd in n for nd in no_decay					):
                  params_without_wd.append(_SCREAMING_SNAKE_CASE					)
            else:
                  params_with_wd.append(_SCREAMING_SNAKE_CASE					)
      return [
          {"params": params_with_wd, "weight_decay": args.weight_decay},
          {"params": params_without_wd, "weight_decay": 0.0},
      ]
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						List[str]  ,					_SCREAMING_SNAKE_CASE :						List[str]					)->Union[str, Any]:
      logger.info(f'''Step {step}: {metrics}'''					)
      if accelerator.is_main_process:
            accelerator.log(_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE					)
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Tuple  ,					_SCREAMING_SNAKE_CASE :						str  ,					_SCREAMING_SNAKE_CASE :						List[Any]					)->Any:
      # TFLOPs formula (from Equation 3 in Section 5.1 of https://arxiv.org/pdf/2104.04473.pdf).
      _lowerCAmelCase			      =	accelerator.unwrap_model(_SCREAMING_SNAKE_CASE					).config
      _lowerCAmelCase			      =	4 if args.gradient_checkpointing else 3
      _lowerCAmelCase			      =	args.train_batch_size * accelerator.state.num_processes * args.gradient_accumulation_steps
      _lowerCAmelCase			      =	2_4 * checkpoint_factor * batch_size * args.seq_length * config_model.n_layer * (config_model.n_embd**2)
      _lowerCAmelCase			      =	factor * (
          1.0
          + (args.seq_length / (6.0 * config_model.n_embd))
          + (tokenizer.vocab_size / (16.0 * config_model.n_layer * config_model.n_embd))
      )
      _lowerCAmelCase			      =	flops_per_iteration / (elapsed_time * accelerator.state.num_processes * (1_0**1_2))
      return tflops
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						str					)->List[Any]:
      model.eval()
      _lowerCAmelCase			      =	[]
      for step, batch in enumerate(_SCREAMING_SNAKE_CASE					):
            with torch.no_grad():
                  _lowerCAmelCase			      =	model(_SCREAMING_SNAKE_CASE  ,					labels=_SCREAMING_SNAKE_CASE					)
            _lowerCAmelCase			      =	outputs.loss.repeat(args.valid_batch_size					)
            losses.append(accelerator.gather(_SCREAMING_SNAKE_CASE					)					)
            if args.max_eval_steps > 0 and step >= args.max_eval_steps:
                  break
      _lowerCAmelCase			      =	torch.cat(_SCREAMING_SNAKE_CASE					)
      _lowerCAmelCase			      =	losses[: eval_dataloader.dataset.current_size].mean()
      try:
            _lowerCAmelCase			      =	torch.exp(_SCREAMING_SNAKE_CASE					)
      except OverflowError:
            _lowerCAmelCase			      =	float('''inf'''					)
      return loss.item(), perplexity.item()


# Settings
UpperCAmelCase_          =				HfArgumentParser(TrainingArguments)
UpperCAmelCase_          =				parser.parse_args()

# Accelerator
UpperCAmelCase_          =				ProjectConfiguration(project_dir=args.save_dir, logging_dir="log")
UpperCAmelCase_          =				Accelerator(log_with=["wandb", "tensorboard"], project_config=config)
UpperCAmelCase_          =				{str(k): str(v) for k, v in accelerator.state.__dict__.items()}

UpperCAmelCase_          =				Namespace(**vars(args), **acc_state)
UpperCAmelCase_          =				accelerator.state.num_processes * args.train_batch_size
set_seed(args.seed)

# Clone model repository
if accelerator.is_main_process:
    UpperCAmelCase_          =				Repository(args.save_dir, clone_from=args.model_ckpt)

# Logging
UpperCAmelCase_      ,						UpperCAmelCase_          =				setup_logging(args)
logger.info(accelerator.state)

# Checkout new branch on repo
if accelerator.is_main_process:
    hf_repo.git_checkout(run_name, create_branch_ok=True)

# Load model and tokenizer
UpperCAmelCase_          =				AutoModelForCausalLM.from_pretrained(args.save_dir)
if args.gradient_checkpointing:
    model.gradient_checkpointing_enable()
UpperCAmelCase_          =				AutoTokenizer.from_pretrained(args.save_dir)

# Load dataset and dataloader
UpperCAmelCase_      ,						UpperCAmelCase_          =				create_dataloaders(args)

# Prepare the optimizer and learning rate scheduler
UpperCAmelCase_          =				AdamW(get_grouped_params(model, args), lr=args.learning_rate)
UpperCAmelCase_          =				get_scheduler(
    name=args.lr_scheduler_type,
    optimizer=optimizer,
    num_warmup_steps=args.num_warmup_steps,
    num_training_steps=args.max_train_steps,
)
accelerator.register_for_checkpointing(lr_scheduler)
def 				UpperCAmelCase__		(					)->List[str]:
      return optimizer.param_groups[0]["lr"]


# Prepare everything with our `accelerator`.
UpperCAmelCase_      ,						UpperCAmelCase_      ,						UpperCAmelCase_      ,						UpperCAmelCase_          =				accelerator.prepare(
    model, optimizer, train_dataloader, eval_dataloader
)

# load in the weights and states from a previous save
if args.resume_from_checkpoint:
    if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
        accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}""")
        accelerator.load_state(args.resume_from_checkpoint)
        UpperCAmelCase_          =				os.path.basename(args.resume_from_checkpoint)
    else:
        # Get the most recent checkpoint
        UpperCAmelCase_          =				[f.name for f in os.scandir(args.save_dir) if f.is_dir() and "step" in str(f)]
        dirs.sort(key=os.path.getctime)
        UpperCAmelCase_          =				dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
    # Extract the step of the checkpoint to continue from there
    UpperCAmelCase_          =				os.path.splitext(path)[0]
    UpperCAmelCase_          =				int(training_difference.replace("step_", ""))

# Train model
model.train()
UpperCAmelCase_          =				0
UpperCAmelCase_          =				time.time()
UpperCAmelCase_          =				0
for step, batch in enumerate(train_dataloader, start=1):
    if args.resume_from_checkpoint and step < resume_step:
        continue  # we need to skip steps until we reach the resumed step
    UpperCAmelCase_          =				model(batch, labels=batch, use_cache=False).loss
    UpperCAmelCase_          =				accelerator.gather(loss.repeat(args.train_batch_size)).mean()
    loss_tracking += avg_loss.item() / args.gradient_accumulation_steps
    log_metrics(step, {"samples": step * samples_per_step, "loss_per_step/train": loss.item()})
    UpperCAmelCase_          =				loss / args.gradient_accumulation_steps
    if step % args.gradient_accumulation_steps != 0:
        # Prevent backward from doing gradient all_reduce in every step
        if accelerator.distributed_type == DistributedType.MULTI_GPU:
            with model.no_sync():
                accelerator.backward(loss)
        else:
            accelerator.backward(loss)
    else:
        UpperCAmelCase_          =				get_lr()
        accelerator.backward(loss)
        accelerator.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        UpperCAmelCase_          =				time.time() - t_start
        UpperCAmelCase_          =				compute_tflops(elapsed_time, accelerator, args)
        log_metrics(
            step,
            {
                "steps": completed_steps,
                "loss/train": loss_tracking,
                "lr": lr,
                "tflops": tflops,
                "time_per_iteration": elapsed_time,
            },
        )
        UpperCAmelCase_          =				time.time()
        UpperCAmelCase_          =				0
        completed_steps += 1
    if step % args.save_checkpoint_steps == 0:
        logger.info("Evaluating and saving model checkpoint")
        UpperCAmelCase_      ,						UpperCAmelCase_          =				evaluate(args)
        log_metrics(step, {"loss/eval": eval_loss, "perplexity": perplexity})
        accelerator.wait_for_everyone()
        UpperCAmelCase_          =				os.path.join(args.save_dir, F"""step_{step}""")
        accelerator.save_state(save_dir)
        if accelerator.is_main_process:
            hf_repo.push_to_hub(commit_message=F"""step {step}""")
        model.train()
    if completed_steps >= args.max_train_steps:
        break

# Evaluate and save the last checkpoint
logger.info("Evaluating and saving model after training")
UpperCAmelCase_      ,						UpperCAmelCase_          =				evaluate(args)
log_metrics(step, {"loss/eval": eval_loss, "perplexity": perplexity})
accelerator.wait_for_everyone()
UpperCAmelCase_          =				accelerator.unwrap_model(model)
unwrapped_model.save_pretrained(args.save_dir, save_function=accelerator.save)
UpperCAmelCase_          =				os.path.join(args.save_dir, F"""step_{step}""")
accelerator.save_state(save_dir)
if accelerator.is_main_process:
    hf_repo.push_to_hub(commit_message="final model")




import argparse
import glob
import inspect
import os
import re
import unittest

from get_test_info import get_test_classes

from tests.test_pipeline_mixin import pipeline_test_mapping


UpperCAmelCase_          =				{}
for task, _ in pipeline_test_mapping.items():
    UpperCAmelCase_          =				{"pt": None, "tf": None}


# DO **NOT** add item to this set (unless the reason is approved)
UpperCAmelCase_          =				{
    "tests/models/esm/test_modeling_esmfold.py",  # The pipeline test mapping is added to `test_modeling_esm.py`
}
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Tuple					)->Union[str, Any]:

      if "ModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
            return "pt"
      elif "TFModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
            return "tf"
      elif "FlaxModelTesterMixin" in [x.__name__ for x in test_class.__bases__]:
            return "flax"
      else:
            return None
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Any  ,					_SCREAMING_SNAKE_CASE :						str					)->Dict:
      # Use the cached results
      if PIPELINE_TEST_MAPPING[task].get(_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE					) is not None:
            return PIPELINE_TEST_MAPPING[task][framework]

      _lowerCAmelCase			      =	pipeline_test_mapping[task]['''test''']
      _lowerCAmelCase			      =	None

      if framework == "pt":
            _lowerCAmelCase			      =	getattr(_SCREAMING_SNAKE_CASE  ,					'''model_mapping'''  ,					_SCREAMING_SNAKE_CASE					)
      elif framework == "tf":
            _lowerCAmelCase			      =	getattr(_SCREAMING_SNAKE_CASE  ,					'''tf_model_mapping'''  ,					_SCREAMING_SNAKE_CASE					)

      if mapping is not None:
            _lowerCAmelCase			      =	dict(mapping.items()					)

      # cache the results
      _lowerCAmelCase			      =	mapping
      return mapping
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Union[str, Any]  ,					_SCREAMING_SNAKE_CASE :						Union[str, Any]					)->List[Any]:
      _lowerCAmelCase			      =	get_framework(_SCREAMING_SNAKE_CASE					)
      if framework is None:
            return None
      _lowerCAmelCase			      =	get_mapping_for_task(_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE					)
      if mapping is None:
            return None

      _lowerCAmelCase			      =	list({model_class.config_class for model_class in test_class.all_model_classes}					)
      if len(_SCREAMING_SNAKE_CASE					) != 1:
            raise ValueError('''There should be exactly one configuration class from `test_class.all_model_classes`.'''					)

      # This could be a list/tuple of model classes, but it's rare.
      _lowerCAmelCase			      =	mapping.get(config_classes[0]  ,					_SCREAMING_SNAKE_CASE					)
      if isinstance(_SCREAMING_SNAKE_CASE  ,					(tuple, list)					):
            _lowerCAmelCase			      =	sorted(_SCREAMING_SNAKE_CASE  ,					key=lambda _SCREAMING_SNAKE_CASE					: x.__name__					)

      return model_class
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						int					)->Dict:
      _lowerCAmelCase			      =	[(task, get_model_for_pipeline_test(_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE					)) for task in pipeline_test_mapping]
      _lowerCAmelCase			      =	sorted([(task, model) for task, model in mapping if model is not None]  ,					key=lambda _SCREAMING_SNAKE_CASE					: x[0]					)

      return dict(_SCREAMING_SNAKE_CASE					)
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Union[str, Any]					)->Dict:
      _lowerCAmelCase			      =	get_framework(_SCREAMING_SNAKE_CASE					)
      if framework == "pt":
            _lowerCAmelCase			      =	'''torch'''
      _lowerCAmelCase			      =	'''{}'''

      _lowerCAmelCase			      =	get_pipeline_model_mapping(_SCREAMING_SNAKE_CASE					)
      if len(_SCREAMING_SNAKE_CASE					) == 0:
            return ""

      _lowerCAmelCase			      =	[]
      for task, model_classes in mapping.items():
            if isinstance(_SCREAMING_SNAKE_CASE  ,					(tuple, list)					):
                  # A list/tuple of model classes
                  _lowerCAmelCase			      =	'''(''' + ''', '''.join([x.__name__ for x in model_classes]					) + ''')'''
            else:
                  # A single model class
                  _lowerCAmelCase			      =	model_classes.__name__
            texts.append(f'''"{task}": {value}'''					)
      _lowerCAmelCase			      =	'''{''' + ''', '''.join(_SCREAMING_SNAKE_CASE					) + '''}'''
      _lowerCAmelCase			      =	f'''pipeline_model_mapping = {text} if is_{framework}_available() else {default_value}'''

      return text
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Optional[Any]					)->int:
      _lowerCAmelCase			      =	{'''ModelTesterMixin''', '''TFModelTesterMixin''', '''FlaxModelTesterMixin'''}
      if not issubclass(_SCREAMING_SNAKE_CASE  ,					unittest.TestCase					):
            return False
      return len(base_class_names.intersection([x.__name__ for x in test_class.__bases__]					)					) > 0
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Any					)->int:
      _lowerCAmelCase			      =	[x for x in get_test_classes(_SCREAMING_SNAKE_CASE					) if is_valid_test_class(_SCREAMING_SNAKE_CASE					)]

      _lowerCAmelCase			      =	None
      for test_class in test_classes:
            # If a test class has defined `pipeline_model_mapping`, let's take it
            if getattr(_SCREAMING_SNAKE_CASE  ,					'''pipeline_model_mapping'''  ,					_SCREAMING_SNAKE_CASE					) is not None:
                  _lowerCAmelCase			      =	test_class
                  break
    # Take the test class with the shortest name (just a heuristic)
      if target_test_class is None and len(_SCREAMING_SNAKE_CASE					) > 0:
            _lowerCAmelCase			      =	sorted(_SCREAMING_SNAKE_CASE  ,					key=lambda _SCREAMING_SNAKE_CASE					: (len(x.__name__					), x.__name__)					)[0]

      return target_test_class
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						List[str]  ,					_SCREAMING_SNAKE_CASE :						Tuple  ,					_SCREAMING_SNAKE_CASE :						Optional[int]					)->Union[str, Any]:
      _lowerCAmelCase			      =	start_idx
      for idx, line in enumerate(lines[start_idx:]					):
            _lowerCAmelCase			      =	len(_SCREAMING_SNAKE_CASE					) - len(line.lstrip()					)
            if idx == 0 or indent > indent_level or (indent == indent_level and line.strip() == ")"):
                  _lowerCAmelCase			      =	start_idx + idx
            elif idx > 0 and indent <= indent_level:
                  # Outside the definition block of `pipeline_model_mapping`
                  break

      return end_idx
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Dict  ,					_SCREAMING_SNAKE_CASE :						Tuple=False					)->Tuple:
      if getattr(_SCREAMING_SNAKE_CASE  ,					'''pipeline_model_mapping'''  ,					_SCREAMING_SNAKE_CASE					) is not None:
            if not overwrite:
                  return "", -1

      _lowerCAmelCase			      =	get_pipeline_model_mapping_string(_SCREAMING_SNAKE_CASE					)
      if len(_SCREAMING_SNAKE_CASE					) == 0:
            return "", -1
      _lowerCAmelCase			      =	line_to_add + '''\n'''

      # The code defined the class `test_class`
      _lowerCAmelCase      , _lowerCAmelCase			      =	inspect.getsourcelines(_SCREAMING_SNAKE_CASE					)
      # `inspect` gives the code for an object, including decorator(s) if any.
      # We (only) need the exact line of the class definition.
      for idx, line in enumerate(_SCREAMING_SNAKE_CASE					):
            if line.lstrip().startswith('''class '''					):
                  _lowerCAmelCase			      =	class_lines[idx:]
                  class_start_line_no += idx
                  break
      _lowerCAmelCase			      =	class_start_line_no + len(_SCREAMING_SNAKE_CASE					) - 1

      # The index in `class_lines` that starts the definition of `all_model_classes`, `all_generative_model_classes` or
      # `pipeline_model_mapping`. This assumes they are defined in such order, and we take the start index of the last
      # block that appears in a `test_class`.
      _lowerCAmelCase			      =	None
      # The indent level of the line at `class_lines[start_idx]` (if defined)
      _lowerCAmelCase			      =	0
      # To record if `pipeline_model_mapping` is found in `test_class`.
      _lowerCAmelCase			      =	None
      for idx, line in enumerate(_SCREAMING_SNAKE_CASE					):
            if line.strip().startswith('''all_model_classes = '''					):
                  _lowerCAmelCase			      =	len(_SCREAMING_SNAKE_CASE					) - len(line.lstrip()					)
                  _lowerCAmelCase			      =	idx
            elif line.strip().startswith('''all_generative_model_classes = '''					):
                  _lowerCAmelCase			      =	len(_SCREAMING_SNAKE_CASE					) - len(line.lstrip()					)
                  _lowerCAmelCase			      =	idx
            elif line.strip().startswith('''pipeline_model_mapping = '''					):
                  _lowerCAmelCase			      =	len(_SCREAMING_SNAKE_CASE					) - len(line.lstrip()					)
                  _lowerCAmelCase			      =	idx
                  _lowerCAmelCase			      =	line
                  break

      if start_idx is None:
            return "", -1
      # Find the ending index (inclusive) of the above found block.
      _lowerCAmelCase			      =	find_block_ending(_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE					)

      # Extract `is_xxx_available()` from existing blocks: some models require specific libraries like `timm` and use
      # `is_timm_available()` instead of `is_torch_available()`.
      # Keep leading and trailing whitespaces
      _lowerCAmelCase			      =	re.compile(r'''\s(is_\S+?_available\(\))\s'''					)
      for line in class_lines[start_idx : end_idx + 1]:
            _lowerCAmelCase			      =	r.search(_SCREAMING_SNAKE_CASE					)
            if backend_condition is not None:
                  # replace the leading and trailing whitespaces to the space character " ".
                  _lowerCAmelCase			      =	''' ''' + backend_condition[0][1:-1] + ''' '''
                  _lowerCAmelCase			      =	r.sub(_SCREAMING_SNAKE_CASE  ,					_SCREAMING_SNAKE_CASE					)
                  break

      if def_line is None:
            # `pipeline_model_mapping` is not defined. The target index is set to the ending index (inclusive) of
            # `all_model_classes` or `all_generative_model_classes`.
            _lowerCAmelCase			      =	end_idx
      else:
            # `pipeline_model_mapping` is defined. The target index is set to be one **BEFORE** its start index.
            _lowerCAmelCase			      =	start_idx - 1
            # mark the lines of the currently existing `pipeline_model_mapping` to be removed.
            for idx in range(_SCREAMING_SNAKE_CASE  ,					end_idx + 1					):
                  # These lines are going to be removed before writing to the test file.
                  _lowerCAmelCase			      =	None  # noqa

    # Make sure the test class is a subclass of `PipelineTesterMixin`.
      _lowerCAmelCase			      =	[x.__name__ for x in test_class.__bases__]
      if "PipelineTesterMixin" not in parent_classes:
            # Put `PipelineTesterMixin` just before `unittest.TestCase`
            _lowerCAmelCase			      =	[x for x in parent_classes if x != '''TestCase'''] + ['''PipelineTesterMixin''']
            if "TestCase" in parent_classes:
                  # Here we **assume** the original string is always with `unittest.TestCase`.
                  _parent_classes.append('''unittest.TestCase'''					)
            _lowerCAmelCase			      =	''', '''.join(_parent_classes					)
            for idx, line in enumerate(_SCREAMING_SNAKE_CASE					):
                  # Find the ending of the declaration of `test_class`
                  if line.strip().endswith('''):'''					):
                        # mark the lines of the declaration of `test_class` to be removed
                        for _idx in range(idx + 1					):
                              _lowerCAmelCase			      =	None  # noqa
                        break
        # Add the new, one-line, class declaration for `test_class`
            _lowerCAmelCase			      =	f'''class {test_class.__name__}({parent_classes}):\n'''

      # Add indentation
      _lowerCAmelCase			      =	''' ''' * indent_level + line_to_add
      # Insert `pipeline_model_mapping` to `class_lines`.
      # (The line at `target_idx` should be kept by definition!)
      _lowerCAmelCase			      =	class_lines[: target_idx + 1] + [line_to_add] + class_lines[target_idx + 1 :]
      # Remove the lines that are marked to be removed
      _lowerCAmelCase			      =	[x for x in class_lines if x is not None]

      # Move from test class to module (in order to write to the test file)
      _lowerCAmelCase			      =	inspect.getsourcelines(inspect.getmodule(_SCREAMING_SNAKE_CASE					)					)[0]
      # Be careful with the 1-off between line numbers and array indices
      _lowerCAmelCase			      =	module_lines[: class_start_line_no - 1] + class_lines + module_lines[class_end_line_no:]
      _lowerCAmelCase			      =	''''''.join(_SCREAMING_SNAKE_CASE					)

      _lowerCAmelCase			      =	inspect.getsourcefile(_SCREAMING_SNAKE_CASE					)
      with open(_SCREAMING_SNAKE_CASE  ,					'''w'''  ,					encoding='''UTF-8'''  ,					newline='''\n'''					) as fp:
            fp.write(_SCREAMING_SNAKE_CASE					)

      return line_to_add
def 				UpperCAmelCase__		(					_SCREAMING_SNAKE_CASE :						Optional[int]  ,					_SCREAMING_SNAKE_CASE :						Optional[Any]=False					)->str:
      _lowerCAmelCase			      =	find_test_class(_SCREAMING_SNAKE_CASE					)
      if test_class:
            add_pipeline_model_mapping(_SCREAMING_SNAKE_CASE  ,					overwrite=_SCREAMING_SNAKE_CASE					)


if __name__ == "__main__":
    UpperCAmelCase_          =				argparse.ArgumentParser()
    parser.add_argument(
        "--test_file", type=str, help="A path to the test file, starting with the repository's `tests` directory."
    )
    parser.add_argument(
        "--all",
        action="store_true",
        help="If to check and modify all test files.",
    )
    parser.add_argument(
        "--overwrite",
        action="store_true",
        help="If to overwrite a test class if it has already defined `pipeline_model_mapping`.",
    )
    UpperCAmelCase_          =				parser.parse_args()

    if not args.all and not args.test_file:
        raise ValueError("Please specify either `test_file` or pass `--all` to check/modify all test files.")
    elif args.all and args.test_file:
        raise ValueError("Only one of `--test_file` and `--all` could be specified.")

    UpperCAmelCase_          =				[]
    if args.test_file:
        UpperCAmelCase_          =				[args.test_file]
    else:
        UpperCAmelCase_          =				os.path.join("tests", "models", "**", "test_modeling_*.py")
        for test_file in glob.glob(pattern):
            # `Flax` is not concerned at this moment
            if not test_file.startswith("test_modeling_flax_"):
                test_files.append(test_file)

    for test_file in test_files:
        if test_file in TEST_FILE_TO_IGNORE:
            print(F"""[SKIPPED] {test_file} is skipped as it is in `TEST_FILE_TO_IGNORE` in the file {__file__}.""")
            continue
        add_pipeline_model_mapping_to_test_file(test_file, overwrite=args.overwrite)