code: 
  
  
  
  import argparse
  import os
  
  import evaluate
  import torch
  from datasets import load_dataset
  from torch.optim import AdamW
  from torch.utils.data import DataLoader
  from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed
  
  from accelerate import Accelerator, DistributedType
  
  
  ########################################################################
  # This is a fully working simple example to use Accelerate
  #
  # This example trains a Bert base model on GLUE MRPC
  # in any of the following settings (with the same script):
  #   - single CPU or single GPU
  #   - multi GPUS (using PyTorch distributed mode)
  #   - (multi) TPUs
  #   - fp16 (mixed-precision) or fp32 (normal precision)
  #
  # This example also demonstrates the checkpointing and sharding capabilities
  #
  # To run it in each of these various modes, follow the instructions
  # in the readme for examples:
  # https://github.com/huggingface/accelerate/tree/main/examples
  #
  ########################################################################
  
  
  __A       					=		16
  __A       					=		32
  
  
  
  def  __a  (					lowerCAmelCase_     :      Optional[Any]    ,lowerCAmelCase_     :      Optional[Any]       )   ->				int:
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						if args.with_tracking:
  												UpperCAmelCase_=     Accelerator(
  												    cpu=args.cpu    ,mixed_precision=args.mixed_precision    ,log_with="""all"""    ,project_dir=args.project_dir       )
  						else:
  												UpperCAmelCase_=     Accelerator(cpu=args.cpu    ,mixed_precision=args.mixed_precision       )
  
  						if hasattr(args.checkpointing_steps    ,"""isdigit"""       ):
  												if args.checkpointing_steps == "epoch":
  																		UpperCAmelCase_=     args.checkpointing_steps
  												elif args.checkpointing_steps.isdigit():
  																		UpperCAmelCase_=     int(args.checkpointing_steps       )
  												else:
  																		raise ValueError(
  																		    F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""       )
  						else:
  												UpperCAmelCase_=     None
  						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  						UpperCAmelCase_=     config["""lr"""]
  						UpperCAmelCase_=     int(config["""num_epochs"""]       )
  						UpperCAmelCase_=     int(config["""seed"""]       )
  						UpperCAmelCase_=     int(config["""batch_size"""]       )
  
  						# We need to initialize the trackers we use, and also store our configuration
  						if args.with_tracking:
  												UpperCAmelCase_=     os.path.split(lowerCAmelCase_       )[-1].split("""."""       )[0]
  												accelerator.init_trackers(lowerCAmelCase_    ,lowerCAmelCase_       )
  
  						UpperCAmelCase_=     AutoTokenizer.from_pretrained("""bert-base-cased"""       )
  						UpperCAmelCase_=     load_dataset("""glue"""    ,"""mrpc"""       )
  						UpperCAmelCase_=     evaluate.load("""glue"""    ,"""mrpc"""       )
  
  						def tokenize_function(lowerCAmelCase_     :      Union[str, Any]       ):
  												# max_length=None => use the model max length (it's actually the default)
  												UpperCAmelCase_=     tokenizer(examples["""sentence1"""]    ,examples["""sentence2"""]    ,truncation=lowerCAmelCase_    ,max_length=lowerCAmelCase_       )
  												return outputs
  
  						# Apply the method we just defined to all the examples in all the splits of the dataset
  						# starting with the main process first:
  						with accelerator.main_process_first():
  												UpperCAmelCase_=     datasets.map(
  												    lowerCAmelCase_    ,batched=lowerCAmelCase_    ,remove_columns=["""idx""", """sentence1""", """sentence2"""]    ,)
  
  						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  						# transformers library
  						UpperCAmelCase_=     tokenized_datasets.rename_column("""label"""    ,"""labels"""       )
  
  						# If the batch size is too big we use gradient accumulation
  						UpperCAmelCase_=     1
  						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
  												UpperCAmelCase_=     batch_size // MAX_GPU_BATCH_SIZE
  												UpperCAmelCase_=     MAX_GPU_BATCH_SIZE
  
  						def collate_fn(lowerCAmelCase_     :      str       ):
  												# On TPU it's best to pad everything to the same length or training will be very slow.
  												UpperCAmelCase_=     1_28 if accelerator.distributed_type == DistributedType.TPU else None
  												# When using mixed precision we want round multiples of 8/16
  												if accelerator.mixed_precision == "fp8":
  																		UpperCAmelCase_=     16
  												elif accelerator.mixed_precision != "no":
  																		UpperCAmelCase_=     8
  												else:
  																		UpperCAmelCase_=     None
  
  												return tokenizer.pad(
  												    lowerCAmelCase_    ,padding="""longest"""    ,max_length=lowerCAmelCase_    ,pad_to_multiple_of=lowerCAmelCase_    ,return_tensors="""pt"""    ,)
  
  						# Instantiate dataloaders.
  						UpperCAmelCase_=     DataLoader(
  						    tokenized_datasets["""train"""]    ,shuffle=lowerCAmelCase_    ,collate_fn=lowerCAmelCase_    ,batch_size=lowerCAmelCase_       )
  						UpperCAmelCase_=     DataLoader(
  						    tokenized_datasets["""validation"""]    ,shuffle=lowerCAmelCase_    ,collate_fn=lowerCAmelCase_    ,batch_size=lowerCAmelCase_       )
  
  						set_seed(lowerCAmelCase_       )
  
  						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
  						UpperCAmelCase_=     AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""    ,return_dict=lowerCAmelCase_       )
  
  						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  						UpperCAmelCase_=     model.to(accelerator.device       )
  
  						# Instantiate optimizer
  						UpperCAmelCase_=     AdamW(params=model.parameters()    ,lr=lowerCAmelCase_       )
  
  						# Instantiate scheduler
  						UpperCAmelCase_=     get_linear_schedule_with_warmup(
  						    optimizer=lowerCAmelCase_    ,num_warmup_steps=1_00    ,num_training_steps=(len(lowerCAmelCase_       ) * num_epochs) // gradient_accumulation_steps    ,)
  
  						# Prepare everything
  						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  						# prepare method.
  						UpperCAmelCase_,     UpperCAmelCase_,     UpperCAmelCase_,     UpperCAmelCase_,     UpperCAmelCase_=     accelerator.prepare(
  						    lowerCAmelCase_    ,lowerCAmelCase_    ,lowerCAmelCase_    ,lowerCAmelCase_    ,lowerCAmelCase_       )
  
  						# We need to keep track of how many total steps we have iterated over
  						UpperCAmelCase_=     0
  						# We also need to keep track of the stating epoch so files are named properly
  						UpperCAmelCase_=     0
  
  						# Potentially load in the weights and states from a previous save
  						if args.resume_from_checkpoint:
  												if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
  																		accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""       )
  																		accelerator.load_state(args.resume_from_checkpoint       )
  																		UpperCAmelCase_=     os.path.basename(args.resume_from_checkpoint       )
  												else:
  																		# Get the most recent checkpoint
  																		UpperCAmelCase_=     [f.name for f in os.scandir(os.getcwd()       ) if f.is_dir()]
  																		dirs.sort(key=os.path.getctime       )
  																		UpperCAmelCase_=     dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
  												# Extract `epoch_{i}` or `step_{i}`
  												UpperCAmelCase_=     os.path.splitext(lowerCAmelCase_       )[0]
  
  												if "epoch" in training_difference:
  																		UpperCAmelCase_=     int(training_difference.replace("""epoch_"""    ,""""""       )       ) + 1
  																		UpperCAmelCase_=     None
  												else:
  																		UpperCAmelCase_=     int(training_difference.replace("""step_"""    ,""""""       )       )
  																		UpperCAmelCase_=     resume_step // len(lowerCAmelCase_       )
  																		resume_step -= starting_epoch * len(lowerCAmelCase_       )
  
      # Now we train the model
  						for epoch in range(lowerCAmelCase_    ,lowerCAmelCase_       ):
  												model.train()
  												if args.with_tracking:
  																		UpperCAmelCase_=     0
  												if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
  																		# We need to skip steps until we reach the resumed step
  																		UpperCAmelCase_=     accelerator.skip_first_batches(lowerCAmelCase_    ,lowerCAmelCase_       )
  																		overall_step += resume_step
  												else:
  																		# After the first iteration though, we need to go back to the original dataloader
  																		UpperCAmelCase_=     train_dataloader
  												for step, batch in enumerate(lowerCAmelCase_       ):
  																		# We could avoid this line since we set the accelerator with `device_placement=True`.
  																		batch.to(accelerator.device       )
  																		UpperCAmelCase_=     model(**lowerCAmelCase_       )
  																		UpperCAmelCase_=     outputs.loss
  																		UpperCAmelCase_=     loss / gradient_accumulation_steps
  																		# We keep track of the loss at each epoch
  																		if args.with_tracking:
  																								total_loss += loss.detach().float()
  																		accelerator.backward(lowerCAmelCase_       )
  																		if step % gradient_accumulation_steps == 0:
  																								optimizer.step()
  																								lr_scheduler.step()
  																								optimizer.zero_grad()
  
  																		overall_step += 1
  
  																		if isinstance(lowerCAmelCase_    ,lowerCAmelCase_       ):
  																								UpperCAmelCase_=     F"""step_{overall_step}"""
  																								if overall_step % checkpointing_steps == 0:
  																														if args.output_dir is not None:
  																																				UpperCAmelCase_=     os.path.join(args.output_dir    ,lowerCAmelCase_       )
  																														accelerator.save_state(lowerCAmelCase_       )
  
  												model.eval()
  												for step, batch in enumerate(lowerCAmelCase_       ):
  																		# We could avoid this line since we set the accelerator with `device_placement=True`.
  																		batch.to(accelerator.device       )
  																		with torch.no_grad():
  																								UpperCAmelCase_=     model(**lowerCAmelCase_       )
  																		UpperCAmelCase_=     outputs.logits.argmax(dim=-1       )
  																		UpperCAmelCase_,     UpperCAmelCase_=     accelerator.gather_for_metrics((predictions, batch["""labels"""])       )
  																		metric.add_batch(
  																		    predictions=lowerCAmelCase_    ,references=lowerCAmelCase_    ,)
  
  												UpperCAmelCase_=     metric.compute()
  												# Use accelerator.print to print only on the main process.
  												accelerator.print(F"""epoch {epoch}:"""    ,lowerCAmelCase_       )
  												if args.with_tracking:
  																		accelerator.log(
  																		    {
  																		        """accuracy""": eval_metric["""accuracy"""],
  																		        """f1""": eval_metric["""f1"""],
  																		        """train_loss""": total_loss.item() / len(lowerCAmelCase_       ),
  																		        """epoch""": epoch,
  																		    }    ,step=lowerCAmelCase_    ,)
  
  												if checkpointing_steps == "epoch":
  																		UpperCAmelCase_=     F"""epoch_{epoch}"""
  																		if args.output_dir is not None:
  																								UpperCAmelCase_=     os.path.join(args.output_dir    ,lowerCAmelCase_       )
  																		accelerator.save_state(lowerCAmelCase_       )
  
  						if args.with_tracking:
  												accelerator.end_training()
  
  
  
  def  __a  (					)   ->				Tuple:
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						UpperCAmelCase_=     argparse.ArgumentParser(description="""Simple example of training script."""       )
  						parser.add_argument(
  						    """--mixed_precision"""    ,type=lowerCAmelCase_    ,default=lowerCAmelCase_    ,choices=["""no""", """fp16""", """bf16""", """fp8"""]    ,help="""Whether to use mixed precision. Choose"""
  						    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
  						    """and an Nvidia Ampere GPU."""    ,)
  						parser.add_argument("""--cpu"""    ,action="""store_true"""    ,help="""If passed, will train on the CPU."""       )
  						parser.add_argument(
  						    """--checkpointing_steps"""    ,type=lowerCAmelCase_    ,default=lowerCAmelCase_    ,help="""Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch."""    ,)
  						parser.add_argument(
  						    """--resume_from_checkpoint"""    ,type=lowerCAmelCase_    ,default=lowerCAmelCase_    ,help="""If the training should continue from a checkpoint folder."""    ,)
  						parser.add_argument(
  						    """--with_tracking"""    ,action="""store_true"""    ,help="""Whether to load in all available experiment trackers from the environment and use them for logging."""    ,)
  						parser.add_argument(
  						    """--output_dir"""    ,type=lowerCAmelCase_    ,default="""."""    ,help="""Optional save directory where all checkpoint folders will be stored. Default is the current working directory."""    ,)
  						parser.add_argument(
  						    """--project_dir"""    ,type=lowerCAmelCase_    ,default="""logs"""    ,help="""Location on where to store experiment tracking logs` and relevent project information"""    ,)
  						UpperCAmelCase_=     parser.parse_args()
  						UpperCAmelCase_=     {"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
  						training_function(lowerCAmelCase_    ,lowerCAmelCase_       )
  
  
  if __name__ == "__main__":
  	main()
  
  
  
  
  
style_context: 
  
  
  
  import unittest
  
  from transformers import YosoConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
  	import torch
  
  	from transformers import (
  	    YosoForMaskedLM,
  	    YosoForMultipleChoice,
  	    YosoForQuestionAnswering,
  	    YosoForSequenceClassification,
  	    YosoForTokenClassification,
  	    YosoModel,
  	)
  	from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  
  
  class 	lowercase		:
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	def __init__(			self			:		int		,						__UpperCAmelCase			:		int		,						__UpperCAmelCase			:		Optional[int]=13		,						__UpperCAmelCase			:		Optional[Any]=7		,						__UpperCAmelCase			:		Union[str, Any]=True		,						__UpperCAmelCase			:		Dict=True		,						__UpperCAmelCase			:		int=True		,						__UpperCAmelCase			:		Optional[int]=True		,						__UpperCAmelCase			:		Optional[Any]=99		,						__UpperCAmelCase			:		List[Any]=32		,						__UpperCAmelCase			:		int=5		,						__UpperCAmelCase			:		List[str]=4		,						__UpperCAmelCase			:		Union[str, Any]=37		,						__UpperCAmelCase			:		Any="gelu"		,						__UpperCAmelCase			:		Any=0.1		,						__UpperCAmelCase			:		List[Any]=0.1		,						__UpperCAmelCase			:		Any=512		,						__UpperCAmelCase			:		Tuple=16		,						__UpperCAmelCase			:		Any=2		,						__UpperCAmelCase			:		Tuple=0.02		,						__UpperCAmelCase			:		Any=3		,						__UpperCAmelCase			:		Tuple=4		,						__UpperCAmelCase			:		int=None		,						)    ->				str:
  							UpperCAmelCase_=     parent
  							UpperCAmelCase_=     batch_size
  							UpperCAmelCase_=     seq_length
  							UpperCAmelCase_=     is_training
  							UpperCAmelCase_=     use_input_mask
  							UpperCAmelCase_=     use_token_type_ids
  							UpperCAmelCase_=     use_labels
  							UpperCAmelCase_=     vocab_size
  							UpperCAmelCase_=     hidden_size
  							UpperCAmelCase_=     num_hidden_layers
  							UpperCAmelCase_=     num_attention_heads
  							UpperCAmelCase_=     intermediate_size
  							UpperCAmelCase_=     hidden_act
  							UpperCAmelCase_=     hidden_dropout_prob
  							UpperCAmelCase_=     attention_probs_dropout_prob
  							UpperCAmelCase_=     max_position_embeddings
  							UpperCAmelCase_=     type_vocab_size
  							UpperCAmelCase_=     type_sequence_label_size
  							UpperCAmelCase_=     initializer_range
  							UpperCAmelCase_=     num_labels
  							UpperCAmelCase_=     num_choices
  							UpperCAmelCase_=     scope
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Tuple  )    ->				str:
  							UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						self.vocab_size  )
  
  							UpperCAmelCase_=     None
  							if self.use_input_mask:
  													UpperCAmelCase_=     random_attention_mask([self.batch_size, self.seq_length]  )
  
  							UpperCAmelCase_=     None
  							if self.use_token_type_ids:
  													UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						self.type_vocab_size  )
  
  							UpperCAmelCase_=     None
  							UpperCAmelCase_=     None
  							UpperCAmelCase_=     None
  							if self.use_labels:
  													UpperCAmelCase_=     ids_tensor([self.batch_size]		,						self.type_sequence_label_size  )
  													UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						self.num_labels  )
  													UpperCAmelCase_=     ids_tensor([self.batch_size]		,						self.num_choices  )
  
  							UpperCAmelCase_=     self.get_config()
  
  							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				List[str]:
  							return YosoConfig(
  							    vocab_size=self.vocab_size		,						hidden_size=self.hidden_size		,						num_hidden_layers=self.num_hidden_layers		,						num_attention_heads=self.num_attention_heads		,						intermediate_size=self.intermediate_size		,						hidden_act=self.hidden_act		,						hidden_dropout_prob=self.hidden_dropout_prob		,						attention_probs_dropout_prob=self.attention_probs_dropout_prob		,						max_position_embeddings=self.max_position_embeddings		,						type_vocab_size=self.type_vocab_size		,						is_decoder=__UpperCAmelCase		,						initializer_range=self.initializer_range		,						)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Union[str, Any]:
  							UpperCAmelCase_=     self.get_config()
  							UpperCAmelCase_=     300
  							return config
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]  )    ->				str:
  							(
  							    (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     
  							)=     self.prepare_config_and_inputs()
  
  							UpperCAmelCase_=     True
  							UpperCAmelCase_=     floats_tensor([self.batch_size, self.seq_length, self.hidden_size]  )
  							UpperCAmelCase_=     ids_tensor([self.batch_size, self.seq_length]		,						vocab_size=2  )
  
  							return (
  							    config,
  							    input_ids,
  							    token_type_ids,
  							    input_mask,
  							    sequence_labels,
  							    token_labels,
  							    choice_labels,
  							    encoder_hidden_states,
  							    encoder_attention_mask,
  							)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		int		,						__UpperCAmelCase			:		Union[str, Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Any  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoModel(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase  )
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase  )
  							UpperCAmelCase_=     model(__UpperCAmelCase  )
  							self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		Dict		,						)    ->				List[str]:
  							UpperCAmelCase_=     True
  							UpperCAmelCase_=     YosoModel(__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						encoder_hidden_states=__UpperCAmelCase		,						encoder_attention_mask=__UpperCAmelCase		,						)
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						encoder_hidden_states=__UpperCAmelCase		,						)
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase  )
  							self.parent.assertEqual(result.last_hidden_state.shape		,						(self.batch_size, self.seq_length, self.hidden_size)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		str  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoForMaskedLM(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase  )
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.vocab_size)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		Optional[int]  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoForQuestionAnswering(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						start_positions=__UpperCAmelCase		,						end_positions=__UpperCAmelCase		,						)
  							self.parent.assertEqual(result.start_logits.shape		,						(self.batch_size, self.seq_length)  )
  							self.parent.assertEqual(result.end_logits.shape		,						(self.batch_size, self.seq_length)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any		,						__UpperCAmelCase			:		Dict		,						__UpperCAmelCase			:		Union[str, Any]		,						__UpperCAmelCase			:		Tuple		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		List[str]  )    ->				str:
  							UpperCAmelCase_=     self.num_labels
  							UpperCAmelCase_=     YosoForSequenceClassification(__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase  )
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_labels)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		int		,						__UpperCAmelCase			:		Union[str, Any]		,						__UpperCAmelCase			:		Optional[Any]		,						__UpperCAmelCase			:		Tuple  )    ->				Union[str, Any]:
  							UpperCAmelCase_=     self.num_labels
  							UpperCAmelCase_=     YosoForTokenClassification(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     model(__UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase  )
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.seq_length, self.num_labels)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str		,						__UpperCAmelCase			:		Optional[int]		,						__UpperCAmelCase			:		List[str]		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		List[Any]		,						__UpperCAmelCase			:		str		,						__UpperCAmelCase			:		Any		,						__UpperCAmelCase			:		Optional[int]  )    ->				Any:
  							UpperCAmelCase_=     self.num_choices
  							UpperCAmelCase_=     YosoForMultipleChoice(config=__UpperCAmelCase  )
  							model.to(__UpperCAmelCase  )
  							model.eval()
  							UpperCAmelCase_=     input_ids.unsqueeze(1  ).expand(-1		,						self.num_choices		,						-1  ).contiguous()
  							UpperCAmelCase_=     token_type_ids.unsqueeze(1  ).expand(-1		,						self.num_choices		,						-1  ).contiguous()
  							UpperCAmelCase_=     input_mask.unsqueeze(1  ).expand(-1		,						self.num_choices		,						-1  ).contiguous()
  							UpperCAmelCase_=     model(
  							    __UpperCAmelCase		,						attention_mask=__UpperCAmelCase		,						token_type_ids=__UpperCAmelCase		,						labels=__UpperCAmelCase		,						)
  							self.parent.assertEqual(result.logits.shape		,						(self.batch_size, self.num_choices)  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Any:
  							UpperCAmelCase_=     self.prepare_config_and_inputs()
  							(
  							    (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     (
  							    UpperCAmelCase_
  							),     
  							)=     config_and_inputs
  							UpperCAmelCase_=     {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
  							return config, inputs_dict
  
  
  
  
  
  
  @require_torch
  class 	lowercase		(				snake_case__				,					snake_case__				,					unittest.TestCase):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	a__   :      Any	     =							(
  	    (
  	        YosoModel,
  	        YosoForMaskedLM,
  	        YosoForMultipleChoice,
  	        YosoForQuestionAnswering,
  	        YosoForSequenceClassification,
  	        YosoForTokenClassification,
  	    )
  	    if is_torch_available()
  	    else ()
  	)
  	a__   :      Optional[int]	     =							False
  	a__   :      Optional[Any]	     =							False
  	a__   :      Dict	     =							False
  
  	a__   :      int	     =							()
  	a__   :      int	     =							(
  	    {
  	        "feature-extraction": YosoModel,
  	        "fill-mask": YosoForMaskedLM,
  	        "question-answering": YosoForQuestionAnswering,
  	        "text-classification": YosoForSequenceClassification,
  	        "token-classification": YosoForTokenClassification,
  	        "zero-shot": YosoForSequenceClassification,
  	    }
  	    if is_torch_available()
  	    else {}
  	)
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Optional[int]:
  							UpperCAmelCase_=     YosoModelTester(self  )
  							UpperCAmelCase_=     ConfigTester(self		,						config_class=__UpperCAmelCase		,						hidden_size=37  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]  )    ->				Union[str, Any]:
  							self.config_tester.run_common_tests()
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		List[str]  )    ->				Dict:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_model(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[int]  )    ->				Any:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							for type in ["absolute", "relative_key", "relative_key_query"]:
  													UpperCAmelCase_=     type
  													self.model_tester.create_and_check_model(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		int  )    ->				Any:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_masked_lm(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				Optional[int]:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_multiple_choice(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Optional[Any]  )    ->				Dict:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_question_answering(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str  )    ->				int:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_sequence_classification(*__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Optional[Any]:
  							UpperCAmelCase_=     self.model_tester.prepare_config_and_inputs()
  							self.model_tester.create_and_check_for_token_classification(*__UpperCAmelCase  )
  
  
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]  )    ->				Optional[Any]:
  							for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
  													UpperCAmelCase_=     YosoModel.from_pretrained(__UpperCAmelCase  )
  													self.assertIsNotNone(__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Tuple:
  							return
  
  
  
  
  
  
  @require_torch
  class 	lowercase		(				unittest.TestCase):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Optional[int]:
  							UpperCAmelCase_=     YosoModel.from_pretrained("""uw-madison/yoso-4096"""  )
  							UpperCAmelCase_=     torch.tensor([[0, 1, 2, 3, 4, 5]]  )
  
  							with torch.no_grad():
  													UpperCAmelCase_=     model(__UpperCAmelCase  )[0]
  
  							UpperCAmelCase_=     torch.Size((1, 6, 768)  )
  							self.assertEqual(output.shape		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     torch.tensor(
  							    [[[-0.0_611, 0.1_242, 0.0_840], [0.0_280, -0.0_048, 0.1_125], [0.0_106, 0.0_226, 0.0_751]]]  )
  
  							self.assertTrue(torch.allclose(output[:, :3, :3]		,						__UpperCAmelCase		,						atol=1E-4  )  )
  
  
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str  )    ->				List[Any]:
  							UpperCAmelCase_=     YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""  )
  							UpperCAmelCase_=     torch.tensor([[0, 1, 2, 3, 4, 5]]  )
  
  							with torch.no_grad():
  													UpperCAmelCase_=     model(__UpperCAmelCase  )[0]
  
  							UpperCAmelCase_=     50_265
  
  							UpperCAmelCase_=     torch.Size((1, 6, vocab_size)  )
  							self.assertEqual(output.shape		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     torch.tensor(
  							    [[[-2.1_313, -3.7_285, -2.2_407], [-2.7_047, -3.3_314, -2.6_408], [0.0_629, -2.5_166, -0.3_356]]]  )
  
  							self.assertTrue(torch.allclose(output[:, :3, :3]		,						__UpperCAmelCase		,						atol=1E-4  )  )
  
  
  
  
  	@slow
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Dict  )    ->				Tuple:
  							UpperCAmelCase_=     YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096"""  )
  							UpperCAmelCase_=     torch.arange(4_096  ).unsqueeze(0  )
  
  							with torch.no_grad():
  													UpperCAmelCase_=     model(__UpperCAmelCase  )[0]
  
  							UpperCAmelCase_=     50_265
  
  							UpperCAmelCase_=     torch.Size((1, 4_096, vocab_size)  )
  							self.assertEqual(output.shape		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     torch.tensor(
  							    [[[-2.3_914, -4.3_742, -5.0_956], [-4.0_988, -4.2_384, -7.0_406], [-3.1_427, -3.7_192, -6.6_800]]]  )
  
  							self.assertTrue(torch.allclose(output[:, :3, :3]		,						__UpperCAmelCase		,						atol=1E-4  )  )
  
  
  
  
  